The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
feat: SAM (segment anything) 3.1 support (CORE-34) (#13408)
Jukka Seppänen749d5b4e8d43
diff --git a/comfy/ldm/sam3/detector.py b/comfy/ldm/sam3/detector.pynew file mode 100644index 000000000..6ae919a79--- /dev/null+++ b/comfy/ldm/sam3/detector.py@@ -0,0 +1,596 @@+# SAM3 detector: transformer encoder-decoder, segmentation head, geometry encoder, scoring.++import math++import torch+import torch.nn as nn+import torch.nn.functional as F+from torchvision.ops import roi_align++from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.sam3.tracker import SAM3Tracker, SAM31Tracker+from comfy.ldm.sam3.sam import SAM3VisionBackbone # noqa: used in __init__+from comfy.ldm.sam3.sam import MLP, PositionEmbeddingSine++TRACKER_CLASSES = {"SAM3": SAM3Tracker, "SAM31": SAM31Tracker}+from comfy.ops import cast_to_input+++def box_cxcywh_to_xyxy(x):+ cx, cy, w, h = x.unbind(-1)+ return torch.stack([cx - 0.5 * w, cy - 0.5 * h, cx + 0.5 * w, cy + 0.5 * h], dim=-1)+++def gen_sineembed_for_position(pos_tensor, num_feats=256):+ """Per-coordinate sinusoidal embedding: (..., N) -> (..., N * num_feats)."""+ assert num_feats % 2 == 0+ hdim = num_feats // 2+ freqs = 10000.0 ** (2 * (torch.arange(hdim, dtype=torch.float32, device=pos_tensor.device) // 2) / hdim)+ embeds = []+ for c in range(pos_tensor.shape[-1]):+ raw = (pos_tensor[..., c].float() * 2 * math.pi).unsqueeze(-1) / freqs+ embeds.append(torch.stack([raw[..., 0::2].sin(), raw[..., 1::2].cos()], dim=-1).flatten(-2))+ return torch.cat(embeds, dim=-1).to(pos_tensor.dtype)+++class SplitMHA(nn.Module):+ """Multi-head attention with separate Q/K/V projections (split from fused in_proj_weight)."""+ def __init__(self, d_model, num_heads=8, device=None, dtype=None, operations=None):+ super().__init__()+ self.num_heads = num_heads+ self.q_proj = operations.Linear(d_model, d_model, device=device, dtype=dtype)+ self.k_proj = operations.Linear(d_model, d_model, device=device, dtype=dtype)+ self.v_proj = operations.Linear(d_model, d_model, device=device, dtype=dtype)+ self.out_proj = operations.Linear(d_model, d_model, device=device, dtype=dtype)++ def forward(self, q_input, k_input=None, v_input=None, mask=None):+ q = self.q_proj(q_input)+ if k_input is None:+ k = self.k_proj(q_input)+ v = self.v_proj(q_input)+ else:+ k = self.k_proj(k_input)+ v = self.v_proj(v_input if v_input is not None else k_input)+ if mask is not None and mask.ndim == 2:+ mask = mask[:, None, None, :] # [B, T] -> [B, 1, 1, T] for SDPA broadcast+ dtype = q.dtype # manual_cast may produce mixed dtypes+ out = optimized_attention(q, k.to(dtype), v.to(dtype), self.num_heads, mask=mask)+ return self.out_proj(out)+++class MLPWithNorm(nn.Module):+ """MLP with residual connection and output LayerNorm."""+ def __init__(self, input_dim, hidden_dim, output_dim, num_layers, residual=True, device=None, dtype=None, operations=None):+ super().__init__()+ dims = [input_dim] + [hidden_dim] * (num_layers - 1) + [output_dim]+ self.layers = nn.ModuleList([+ operations.Linear(dims[i], dims[i + 1], device=device, dtype=dtype)+ for i in range(num_layers)+ ])+ self.out_norm = operations.LayerNorm(output_dim, device=device, dtype=dtype)+ self.residual = residual and (input_dim == output_dim)++ def forward(self, x):+ orig = x+ for i, layer in enumerate(self.layers):+ x = layer(x)+ if i < len(self.layers) - 1:+ x = F.relu(x)+ if self.residual:+ x = x + orig+ return self.out_norm(x)+++class EncoderLayer(nn.Module):+ def __init__(self, d_model=256, num_heads=8, dim_ff=2048, device=None, dtype=None, operations=None):+ super().__init__()+ self.self_attn = SplitMHA(d_model, num_heads, device=device, dtype=dtype, operations=operations)+ self.cross_attn_image = SplitMHA(d_model, num_heads, device=device, dtype=dtype, operations=operations)+ self.linear1 = operations.Linear(d_model, dim_ff, device=device, dtype=dtype)+ self.linear2 = operations.Linear(dim_ff, d_model, device=device, dtype=dtype)+ self.norm1 = operations.LayerNorm(d_model, device=device, dtype=dtype)+ self.norm2 = operations.LayerNorm(d_model, device=device, dtype=dtype)+ self.norm3 = operations.LayerNorm(d_model, device=device, dtype=dtype)++ def forward(self, x, pos, text_memory=None, text_mask=None):+ normed = self.norm1(x)+ q_k = normed + pos+ x = x + self.self_attn(q_k, q_k, normed)+ if text_memory is not None:+ normed = self.norm2(x)+ x = x + self.cross_attn_image(normed, text_memory, text_memory, mask=text_mask)+ normed = self.norm3(x)+ x = x + self.linear2(F.relu(self.linear1(normed)))+ return x+++class TransformerEncoder(nn.Module):+ """Checkpoint: transformer.encoder.layers.N.*"""+ def __init__(self, d_model=256, num_heads=8, dim_ff=2048, num_layers=6, device=None, dtype=None, operations=None):+ super().__init__()+ self.layers = nn.ModuleList([+ EncoderLayer(d_model, num_heads, dim_ff, device=device, dtype=dtype, operations=operations)+ for _ in range(num_layers)+ ])++ def forward(self, x, pos, text_memory=None, text_mask=None):+ for layer in self.layers:+ x = layer(x, pos, text_memory, text_mask)+ return x+++class DecoderLayer(nn.Module):+ def __init__(self, d_model=256, num_heads=8, dim_ff=2048, device=None, dtype=None, operations=None):+ super().__init__()+ self.self_attn = SplitMHA(d_model, num_heads, device=device, dtype=dtype, operations=operations)+ self.cross_attn = SplitMHA(d_model, num_heads, device=device, dtype=dtype, operations=operations)+ self.ca_text = SplitMHA(d_model, num_heads, device=device, dtype=dtype, operations=operations)+ self.norm1 = operations.LayerNorm(d_model, device=device, dtype=dtype)+ self.norm2 = operations.LayerNorm(d_model, device=device, dtype=dtype)+ self.norm3 = operations.LayerNorm(d_model, device=device, dtype=dtype)+ self.catext_norm = operations.LayerNorm(d_model, device=device, dtype=dtype)+ self.linear1 = operations.Linear(d_model, dim_ff, device=device, dtype=dtype)+ self.linear2 = operations.Linear(dim_ff, d_model, device=device, dtype=dtype)++ def forward(self, x, memory, x_pos, memory_pos, text_memory=None, text_mask=None, cross_attn_bias=None):+ q_k = x + x_pos+ x = self.norm2(x + self.self_attn(q_k, q_k, x))+ if text_memory is not None:+ x = self.catext_norm(x + self.ca_text(x + x_pos, text_memory, text_memory, mask=text_mask))+ x = self.norm1(x + self.cross_attn(x + x_pos, memory + memory_pos, memory, mask=cross_attn_bias))+ x = self.norm3(x + self.linear2(F.relu(self.linear1(x))))+ return x+++class TransformerDecoder(nn.Module):+ def __init__(self, d_model=256, num_heads=8, dim_ff=2048, num_layers=6,+ num_queries=200, device=None, dtype=None, operations=None):+ super().__init__()+ self.d_model = d_model+ self.num_queries = num_queries++ self.layers = nn.ModuleList([+ DecoderLayer(d_model, num_heads, dim_ff, device=device, dtype=dtype, operations=operations)+ for _ in range(num_layers)+ ])+ self.norm = operations.LayerNorm(d_model, device=device, dtype=dtype)+ self.query_embed = operations.Embedding(num_queries, d_model, device=device, dtype=dtype)+ self.reference_points = operations.Embedding(num_queries, 4, device=device, dtype=dtype) # Reference points: Embedding(num_queries, 4) — learned anchor boxes+ self.ref_point_head = MLP(d_model * 2, d_model, d_model, 2, device=device, dtype=dtype, operations=operations) # ref_point_head input: 512 (4 coords * 128 sine features each)+ self.bbox_embed = MLP(d_model, d_model, 4, 3, device=device, dtype=dtype, operations=operations)++ self.boxRPB_embed_x = MLP(2, d_model, num_heads, 2, device=device, dtype=dtype, operations=operations)+ self.boxRPB_embed_y = MLP(2, d_model, num_heads, 2, device=device, dtype=dtype, operations=operations)++ self.presence_token = operations.Embedding(1, d_model, device=device, dtype=dtype)+ self.presence_token_head = MLP(d_model, d_model, 1, 3, device=device, dtype=dtype, operations=operations)+ self.presence_token_out_norm = operations.LayerNorm(d_model, device=device, dtype=dtype)++ @staticmethod+ def _inverse_sigmoid(x):+ return torch.log(x / (1 - x + 1e-6) + 1e-6)++ def _compute_box_rpb(self, ref_points, H, W):+ """Box rotary position bias: (B, Q, 4) cxcywh -> (B, n_heads, Q+1, H*W) bias."""+ boxes_xyxy = box_cxcywh_to_xyxy(ref_points)+ B, Q, _ = boxes_xyxy.shape+ coords_h = torch.arange(H, device=ref_points.device, dtype=torch.float32) / H+ coords_w = torch.arange(W, device=ref_points.device, dtype=torch.float32) / W+ deltas_x = coords_w.view(1, 1, -1, 1) - boxes_xyxy[:, :, None, 0:3:2]+ deltas_y = coords_h.view(1, 1, -1, 1) - boxes_xyxy[:, :, None, 1:4:2]++ log2_8 = float(math.log2(8))+ def log_scale(d):+ return torch.sign(d * 8) * torch.log2(torch.abs(d * 8) + 1.0) / log2_8++ rpb_x = self.boxRPB_embed_x(log_scale(deltas_x).to(ref_points.dtype))+ rpb_y = self.boxRPB_embed_y(log_scale(deltas_y).to(ref_points.dtype))++ bias = (rpb_y.unsqueeze(3) + rpb_x.unsqueeze(2)).flatten(2, 3).permute(0, 3, 1, 2)+ pres_bias = torch.zeros(B, bias.shape[1], 1, bias.shape[3], device=bias.device, dtype=bias.dtype)+ return torch.cat([pres_bias, bias], dim=2)++ def forward(self, memory, memory_pos, text_memory=None, text_mask=None, H=72, W=72):+ B = memory.shape[0]+ tgt = cast_to_input(self.query_embed.weight, memory).unsqueeze(0).expand(B, -1, -1)+ presence_out = cast_to_input(self.presence_token.weight, memory)[None].expand(B, -1, -1)+ ref_points = cast_to_input(self.reference_points.weight, memory).unsqueeze(0).expand(B, -1, -1).sigmoid()++ for layer_idx, layer in enumerate(self.layers):+ query_pos = self.ref_point_head(gen_sineembed_for_position(ref_points, self.d_model))+ tgt_with_pres = torch.cat([presence_out, tgt], dim=1)+ pos_with_pres = torch.cat([torch.zeros_like(presence_out), query_pos], dim=1)+ tgt_with_pres = layer(tgt_with_pres, memory, pos_with_pres, memory_pos,+ text_memory, text_mask, self._compute_box_rpb(ref_points, H, W))+ presence_out, tgt = tgt_with_pres[:, :1], tgt_with_pres[:, 1:]+ if layer_idx < len(self.layers) - 1:+ ref_inv = self._inverse_sigmoid(ref_points)+ ref_points = (ref_inv + self.bbox_embed(self.norm(tgt))).sigmoid().detach()++ query_out = self.norm(tgt)+ ref_inv = self._inverse_sigmoid(ref_points)+ boxes = (ref_inv + self.bbox_embed(query_out)).sigmoid()+ presence = self.presence_token_head(self.presence_token_out_norm(presence_out)).squeeze(-1)+ return {"decoder_output": query_out, "pred_boxes": boxes, "presence": presence}+++class Transformer(nn.Module):+ def __init__(self, d_model=256, num_heads=8, dim_ff=2048, enc_layers=6, dec_layers=6,+ num_queries=200, device=None, dtype=None, operations=None):+ super().__init__()+ self.encoder = TransformerEncoder(d_model, num_heads, dim_ff, enc_layers, device=device, dtype=dtype, operations=operations)+ self.decoder = TransformerDecoder(d_model, num_heads, dim_ff, dec_layers, num_queries, device=device, dtype=dtype, operations=operations)+++class GeometryEncoder(nn.Module):+ def __init__(self, d_model=256, num_heads=8, num_layers=3, roi_size=7, device=None, dtype=None, operations=None):+ super().__init__()+ self.d_model = d_model+ self.roi_size = roi_size+ self.pos_enc = PositionEmbeddingSine(num_pos_feats=d_model, normalize=True)+ self.points_direct_project = operations.Linear(2, d_model, device=device, dtype=dtype)+ self.points_pool_project = operations.Linear(d_model, d_model, device=device, dtype=dtype)+ self.points_pos_enc_project = operations.Linear(d_model, d_model, device=device, dtype=dtype)+ self.boxes_direct_project = operations.Linear(4, d_model, device=device, dtype=dtype)+ self.boxes_pool_project = operations.Conv2d(d_model, d_model, kernel_size=roi_size, device=device, dtype=dtype)+ self.boxes_pos_enc_project = operations.Linear(d_model + 2, d_model, device=device, dtype=dtype)+ self.label_embed = operations.Embedding(2, d_model, device=device, dtype=dtype)+ self.cls_embed = operations.Embedding(1, d_model, device=device, dtype=dtype)+ self.norm = operations.LayerNorm(d_model, device=device, dtype=dtype)+ self.img_pre_norm = operations.LayerNorm(d_model, device=device, dtype=dtype)+ self.encode = nn.ModuleList([+ EncoderLayer(d_model, num_heads, 2048, device=device, dtype=dtype, operations=operations)+ for _ in range(num_layers)+ ])+ self.encode_norm = operations.LayerNorm(d_model, device=device, dtype=dtype)+ self.final_proj = operations.Linear(d_model, d_model, device=device, dtype=dtype)++ def _encode_points(self, coords, labels, img_feat_2d):+ """Encode point prompts: direct + pool + pos_enc + label. coords: [B, N, 2] normalized."""+ B, N, _ = coords.shape+ embed = self.points_direct_project(coords)+ # Pool features from backbone at point locations via grid_sample+ grid = (coords * 2 - 1).unsqueeze(2) # [B, N, 1, 2] in [-1, 1]+ sampled = F.grid_sample(img_feat_2d, grid, align_corners=False) # [B, C, N, 1]+ embed = embed + self.points_pool_project(sampled.squeeze(-1).permute(0, 2, 1)) # [B, N, C]+ # Positional encoding of coordinates+ x, y = coords[:, :, 0], coords[:, :, 1] # [B, N]+ pos_x, pos_y = self.pos_enc._encode_xy(x.flatten(), y.flatten())+ enc = torch.cat([pos_x, pos_y], dim=-1).view(B, N, -1)+ embed = embed + self.points_pos_enc_project(cast_to_input(enc, embed))+ embed = embed + cast_to_input(self.label_embed(labels.long()), embed)+ return embed++ def _encode_boxes(self, boxes, labels, img_feat_2d):+ """Encode box prompts: direct + pool + pos_enc + label. boxes: [B, N, 4] normalized cxcywh."""+ B, N, _ = boxes.shape+ embed = self.boxes_direct_project(boxes)+ # ROI align from backbone at box regions+ H, W = img_feat_2d.shape[-2:]+ boxes_xyxy = box_cxcywh_to_xyxy(boxes)+ scale = torch.tensor([W, H, W, H], dtype=boxes_xyxy.dtype, device=boxes_xyxy.device)+ boxes_scaled = boxes_xyxy * scale+ sampled = roi_align(img_feat_2d, boxes_scaled.view(-1, 4).split(N), self.roi_size)+ proj = self.boxes_pool_project(sampled).view(B, N, -1) # Conv2d(roi_size) -> [B*N, C, 1, 1] -> [B, N, C]+ embed = embed + proj+ # Positional encoding of box center + size+ cx, cy, w, h = boxes[:, :, 0], boxes[:, :, 1], boxes[:, :, 2], boxes[:, :, 3]+ enc = self.pos_enc.encode_boxes(cx.flatten(), cy.flatten(), w.flatten(), h.flatten())+ enc = enc.view(B, N, -1)+ embed = embed + self.boxes_pos_enc_project(cast_to_input(enc, embed))+ embed = embed + cast_to_input(self.label_embed(labels.long()), embed)+ return embed++ def forward(self, points=None, boxes=None, image_features=None):+ """Encode geometry prompts. image_features: [B, HW, C] flattened backbone features."""+ # Prepare 2D image features for pooling+ img_feat_2d = None+ if image_features is not None:+ B = image_features.shape[0]+ HW, C = image_features.shape[1], image_features.shape[2]+ hw = int(math.sqrt(HW))+ img_normed = self.img_pre_norm(image_features)+ img_feat_2d = img_normed.permute(0, 2, 1).view(B, C, hw, hw)++ embeddings = []+ if points is not None:+ coords, labels = points+ embeddings.append(self._encode_points(coords, labels, img_feat_2d))+ if boxes is not None:+ B = boxes.shape[0]+ box_labels = torch.ones(B, boxes.shape[1], dtype=torch.long, device=boxes.device)+ embeddings.append(self._encode_boxes(boxes, box_labels, img_feat_2d))+ if not embeddings:+ return None+ geo = torch.cat(embeddings, dim=1)+ geo = self.norm(geo)+ if image_features is not None:+ for layer in self.encode:+ geo = layer(geo, torch.zeros_like(geo), image_features)+ geo = self.encode_norm(geo)+ return self.final_proj(geo)+++class PixelDecoder(nn.Module):+ """Top-down FPN pixel decoder with GroupNorm + ReLU + nearest interpolation."""+ def __init__(self, d_model=256, num_stages=3, device=None, dtype=None, operations=None):+ super().__init__()+ self.conv_layers = nn.ModuleList([operations.Conv2d(d_model, d_model, kernel_size=3, padding=1, device=device, dtype=dtype) for _ in range(num_stages)])+ self.norms = nn.ModuleList([operations.GroupNorm(8, d_model, device=device, dtype=dtype) for _ in range(num_stages)])++ def forward(self, backbone_features):+ prev = backbone_features[-1]+ for i, feat in enumerate(backbone_features[:-1][::-1]):+ prev = F.relu(self.norms[i](self.conv_layers[i](feat + F.interpolate(prev, size=feat.shape[-2:], mode="nearest"))))+ return prev+++class MaskPredictor(nn.Module):+ def __init__(self, d_model=256, device=None, dtype=None, operations=None):+ super().__init__()+ self.mask_embed = MLP(d_model, d_model, d_model, 3, device=device, dtype=dtype, operations=operations)++ def forward(self, query_embeddings, pixel_features):+ mask_embed = self.mask_embed(query_embeddings)+ return torch.einsum("bqc,bchw->bqhw", mask_embed, pixel_features)+++class SegmentationHead(nn.Module):+ def __init__(self, d_model=256, num_heads=8, device=None, dtype=None, operations=None):+ super().__init__()+ self.d_model = d_model+ self.pixel_decoder = PixelDecoder(d_model, 3, device=device, dtype=dtype, operations=operations)+ self.mask_predictor = MaskPredictor(d_model, device=device, dtype=dtype, operations=operations)+ self.cross_attend_prompt = SplitMHA(d_model, num_heads, device=device, dtype=dtype, operations=operations)+ self.cross_attn_norm = operations.LayerNorm(d_model, device=device, dtype=dtype)+ self.instance_seg_head = operations.Conv2d(d_model, d_model, kernel_size=1, device=device, dtype=dtype)+ self.semantic_seg_head = operations.Conv2d(d_model, 1, kernel_size=1, device=device, dtype=dtype)++ def forward(self, query_embeddings, backbone_features, encoder_hidden_states=None, prompt=None, prompt_mask=None):+ if encoder_hidden_states is not None and prompt is not None:+ enc_normed = self.cross_attn_norm(encoder_hidden_states)+ enc_cross = self.cross_attend_prompt(enc_normed, prompt, prompt, mask=prompt_mask)+ encoder_hidden_states = enc_cross + encoder_hidden_states++ if encoder_hidden_states is not None:+ B, H, W = encoder_hidden_states.shape[0], backbone_features[-1].shape[-2], backbone_features[-1].shape[-1]+ encoder_visual = encoder_hidden_states[:, :H * W].permute(0, 2, 1).view(B, self.d_model, H, W)+ backbone_features = list(backbone_features)+ backbone_features[-1] = encoder_visual++ pixel_features = self.pixel_decoder(backbone_features)+ instance_features = self.instance_seg_head(pixel_features)+ masks = self.mask_predictor(query_embeddings, instance_features)+ return masks+++class DotProductScoring(nn.Module):+ def __init__(self, d_model=256, device=None, dtype=None, operations=None):+ super().__init__()+ self.hs_proj = operations.Linear(d_model, d_model, device=device, dtype=dtype)+ self.prompt_proj = operations.Linear(d_model, d_model, device=device, dtype=dtype)+ self.prompt_mlp = MLPWithNorm(d_model, 2048, d_model, 2, device=device, dtype=dtype, operations=operations)+ self.scale = 1.0 / (d_model ** 0.5)++ def forward(self, query_embeddings, prompt_embeddings, prompt_mask=None):+ prompt = self.prompt_mlp(prompt_embeddings)+ if prompt_mask is not None:+ weight = prompt_mask.unsqueeze(-1).to(dtype=prompt.dtype)+ pooled = (prompt * weight).sum(dim=1) / weight.sum(dim=1).clamp(min=1)+ else:+ pooled = prompt.mean(dim=1)+ hs = self.hs_proj(query_embeddings)+ pp = self.prompt_proj(pooled).unsqueeze(-1).to(hs.dtype)+ scores = torch.matmul(hs, pp)+ return (scores * self.scale).clamp(-12.0, 12.0).squeeze(-1)+++class SAM3Detector(nn.Module):+ def __init__(self, d_model=256, embed_dim=1024, num_queries=200, device=None, dtype=None, operations=None, **kwargs):+ super().__init__()+ image_model = kwargs.pop("image_model", "SAM3")+ for k in ("num_heads", "num_head_channels"):+ kwargs.pop(k, None)+ multiplex = image_model == "SAM31"+ # SAM3: 4 FPN levels, drop last (scalp=1); SAM3.1: 3 levels, use all (scalp=0)+ self.scalp = 0 if multiplex else 1+ self.backbone = nn.ModuleDict({+ "vision_backbone": SAM3VisionBackbone(embed_dim=embed_dim, d_model=d_model, multiplex=multiplex, device=device, dtype=dtype, operations=operations, **kwargs),+ "language_backbone": nn.ModuleDict({"resizer": operations.Linear(embed_dim, d_model, device=device, dtype=dtype)}),+ })+ self.transformer = Transformer(d_model=d_model, num_queries=num_queries, device=device, dtype=dtype, operations=operations)+ self.segmentation_head = SegmentationHead(d_model=d_model, device=device, dtype=dtype, operations=operations)+ self.geometry_encoder = GeometryEncoder(d_model=d_model, device=device, dtype=dtype, operations=operations)+ self.dot_prod_scoring = DotProductScoring(d_model=d_model, device=device, dtype=dtype, operations=operations)++ def _get_backbone_features(self, images):+ """Run backbone and return (detector_features, detector_positions, tracker_features, tracker_positions)."""+ bb = self.backbone["vision_backbone"]+ if bb.multiplex:+ all_f, all_p, tf, tp = bb(images, tracker_mode="propagation")+ else:+ all_f, all_p, tf, tp = bb(images, need_tracker=True)+ return all_f, all_p, tf, tp++ @staticmethod+ def _run_geo_layer(layer, x, memory, memory_pos):+ x = x + layer.self_attn(layer.norm1(x))+ x = x + layer.cross_attn_image(layer.norm2(x), memory + memory_pos, memory)+ x = x + layer.linear2(F.relu(layer.linear1(layer.norm3(x))))+ return x++ def _detect(self, features, positions, text_embeddings=None, text_mask=None,+ points=None, boxes=None):+ """Shared detection: geometry encoding, transformer, scoring, segmentation."""+ B = features[0].shape[0]+ # Scalp for encoder (use top-level feature), but keep all levels for segmentation head+ seg_features = features+ if self.scalp > 0:+ features = features[:-self.scalp]+ positions = positions[:-self.scalp]+ enc_feat, enc_pos = features[-1], positions[-1]+ _, _, H, W = enc_feat.shape+ img_flat = enc_feat.flatten(2).permute(0, 2, 1)+ pos_flat = enc_pos.flatten(2).permute(0, 2, 1)++ has_prompts = text_embeddings is not None or points is not None or boxes is not None+ if has_prompts:+ geo_enc = self.geometry_encoder+ geo_prompts = geo_enc(points=points, boxes=boxes, image_features=img_flat)+ geo_cls = geo_enc.norm(geo_enc.final_proj(cast_to_input(geo_enc.cls_embed.weight, img_flat).view(1, 1, -1).expand(B, -1, -1)))+ for layer in geo_enc.encode:+ geo_cls = self._run_geo_layer(layer, geo_cls, img_flat, pos_flat)+ geo_cls = geo_enc.encode_norm(geo_cls)+ if text_embeddings is not None and text_embeddings.shape[0] != B:+ text_embeddings = text_embeddings.expand(B, -1, -1)+ if text_mask is not None and text_mask.shape[0] != B:+ text_mask = text_mask.expand(B, -1)+ parts = [t for t in [text_embeddings, geo_prompts, geo_cls] if t is not None]+ text_embeddings = torch.cat(parts, dim=1)+ n_new = text_embeddings.shape[1] - (text_mask.shape[1] if text_mask is not None else 0)+ if text_mask is not None:+ text_mask = torch.cat([text_mask, torch.ones(B, n_new, dtype=torch.bool, device=text_mask.device)], dim=1)+ else:+ text_mask = torch.ones(B, text_embeddings.shape[1], dtype=torch.bool, device=text_embeddings.device)++ memory = self.transformer.encoder(img_flat, pos_flat, text_embeddings, text_mask)+ dec_out = self.transformer.decoder(memory, pos_flat, text_embeddings, text_mask, H, W)+ query_out, pred_boxes = dec_out["decoder_output"], dec_out["pred_boxes"]++ if text_embeddings is not None:+ scores = self.dot_prod_scoring(query_out, text_embeddings, text_mask)+ else:+ scores = torch.zeros(B, query_out.shape[1], device=query_out.device)++ masks = self.segmentation_head(query_out, seg_features, encoder_hidden_states=memory, prompt=text_embeddings, prompt_mask=text_mask)+ return box_cxcywh_to_xyxy(pred_boxes), scores, masks, dec_out++ def forward(self, images, text_embeddings=None, text_mask=None, points=None, boxes=None, threshold=0.3, orig_size=None):+ features, positions, _, _ = self._get_backbone_features(images)++ if text_embeddings is not None:+ text_embeddings = self.backbone["language_backbone"]["resizer"](text_embeddings)+ if text_mask is not None:+ text_mask = text_mask.bool()++ boxes_xyxy, scores, masks, dec_out = self._detect(+ features, positions, text_embeddings, text_mask, points, boxes)++ if orig_size is not None:+ oh, ow = orig_size+ boxes_xyxy = boxes_xyxy * torch.tensor([ow, oh, ow, oh], device=boxes_xyxy.device, dtype=boxes_xyxy.dtype)+ masks = F.interpolate(masks, size=orig_size, mode="bilinear", align_corners=False)++ return {+ "boxes": boxes_xyxy,+ "scores": scores,+ "masks": masks,+ "presence": dec_out.get("presence"),+ }++ def forward_from_trunk(self, trunk_out, text_embeddings, text_mask):+ """Run detection using a pre-computed ViTDet trunk output.++ text_embeddings must already be resized through language_backbone.resizer.+ Returns dict with boxes (normalized xyxy), scores, masks at detector resolution.+ """+ bb = self.backbone["vision_backbone"]+ features = [conv(trunk_out) for conv in bb.convs]+ positions = [cast_to_input(bb.position_encoding(f), f) for f in features]++ if text_mask is not None:+ text_mask = text_mask.bool()++ boxes_xyxy, scores, masks, _ = self._detect(features, positions, text_embeddings, text_mask)+ return {"boxes": boxes_xyxy, "scores": scores, "masks": masks}+++class SAM3Model(nn.Module):+ def __init__(self, device=None, dtype=None, operations=None, **kwargs):+ super().__init__()+ self.dtype = dtype+ image_model = kwargs.get("image_model", "SAM3")+ tracker_cls = TRACKER_CLASSES[image_model]+ self.detector = SAM3Detector(device=device, dtype=dtype, operations=operations, **kwargs)+ self.tracker = tracker_cls(device=device, dtype=dtype, operations=operations, **kwargs)++ def forward(self, images, **kwargs):+ return self.detector(images, **kwargs)++ def forward_segment(self, images, point_inputs=None, box_inputs=None, mask_inputs=None):+ """Interactive segmentation using SAM decoder with point/box/mask prompts.++ Args:+ images: [B, 3, 1008, 1008] preprocessed images+ point_inputs: {"point_coords": [B, N, 2], "point_labels": [B, N]} in 1008x1008 pixel space+ box_inputs: [B, 2, 2] box corners (top-left, bottom-right) in 1008x1008 pixel space+ mask_inputs: [B, 1, H, W] coarse mask logits to refine+ Returns:+ [B, 1, image_size, image_size] high-res mask logits+ """+ bb = self.detector.backbone["vision_backbone"]+ if bb.multiplex:+ _, _, tracker_features, tracker_positions = bb(images, tracker_mode="interactive")+ else:+ _, _, tracker_features, tracker_positions = bb(images, need_tracker=True)+ if self.detector.scalp > 0:+ tracker_features = tracker_features[:-self.detector.scalp]+ tracker_positions = tracker_positions[:-self.detector.scalp]++ high_res = list(tracker_features[:-1])+ backbone_feat = tracker_features[-1]+ B, C, H, W = backbone_feat.shape+ # Add no-memory embedding (init frame path)+ no_mem = getattr(self.tracker, 'interactivity_no_mem_embed', None)+ if no_mem is None:+ no_mem = getattr(self.tracker, 'no_mem_embed', None)+ if no_mem is not None:+ feat_flat = backbone_feat.flatten(2).permute(0, 2, 1)+ feat_flat = feat_flat + cast_to_input(no_mem, feat_flat)+ backbone_feat = feat_flat.view(B, H, W, C).permute(0, 3, 1, 2)++ num_pts = 0 if point_inputs is None else point_inputs["point_labels"].size(1)+ _, high_res_masks, _, _ = self.tracker._forward_sam_heads(+ backbone_features=backbone_feat,+ point_inputs=point_inputs,+ mask_inputs=mask_inputs,+ box_inputs=box_inputs,+ high_res_features=high_res,+ multimask_output=(0 < num_pts <= 1),+ )+ return high_res_masks++ def forward_video(self, images, initial_masks, pbar=None, text_prompts=None,+ new_det_thresh=0.5, max_objects=0, detect_interval=1):+ """Track video with optional per-frame text-prompted detection."""+ bb = self.detector.backbone["vision_backbone"]++ def backbone_fn(frame, frame_idx=None):+ trunk_out = bb.trunk(frame)+ if bb.multiplex:+ _, _, tf, tp = bb(frame, tracker_mode="propagation", cached_trunk=trunk_out, tracker_only=True)+ else:+ _, _, tf, tp = bb(frame, need_tracker=True, cached_trunk=trunk_out, tracker_only=True)+ return tf, tp, trunk_out++ detect_fn = None+ if text_prompts:+ resizer = self.detector.backbone["language_backbone"]["resizer"]+ resized = [(resizer(emb), m.bool() if m is not None else None) for emb, m in text_prompts]+ def detect_fn(trunk_out):+ all_scores, all_masks = [], []+ for emb, mask in resized:+ det = self.detector.forward_from_trunk(trunk_out, emb, mask)+ all_scores.append(det["scores"])+ all_masks.append(det["masks"])+ return {"scores": torch.cat(all_scores, dim=1), "masks": torch.cat(all_masks, dim=1)}++ if hasattr(self.tracker, 'track_video_with_detection'):+ return self.tracker.track_video_with_detection(+ backbone_fn, images, initial_masks, detect_fn,+ new_det_thresh=new_det_thresh, max_objects=max_objects,+ detect_interval=detect_interval, backbone_obj=bb, pbar=pbar)+ # SAM3 (non-multiplex) — no detection support, requires initial masks+ if initial_masks is None:+ raise ValueError("SAM3 (non-multiplex) requires initial_mask for video tracking")+ return self.tracker.track_video(backbone_fn, images, initial_masks, pbar=pbar, backbone_obj=bb)diff --git a/comfy/ldm/sam3/sam.py b/comfy/ldm/sam3/sam.pynew file mode 100644index 000000000..272781d45--- /dev/null+++ b/comfy/ldm/sam3/sam.py@@ -0,0 +1,425 @@+# SAM3 shared components: primitives, ViTDet backbone, FPN neck, position encodings.++import math++import torch+import torch.nn as nn+import torch.nn.functional as F++from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.flux.math import apply_rope+from comfy.ldm.flux.layers import EmbedND+from comfy.ops import cast_to_input+++class MLP(nn.Module):+ def __init__(self, input_dim, hidden_dim, output_dim, num_layers, sigmoid_output=False, device=None, dtype=None, operations=None):+ super().__init__()+ dims = [input_dim] + [hidden_dim] * (num_layers - 1) + [output_dim]+ self.layers = nn.ModuleList([operations.Linear(dims[i], dims[i + 1], device=device, dtype=dtype) for i in range(num_layers)])+ self.sigmoid_output = sigmoid_output++ def forward(self, x):+ for i, layer in enumerate(self.layers):+ x = F.relu(layer(x)) if i < len(self.layers) - 1 else layer(x)+ return torch.sigmoid(x) if self.sigmoid_output else x+++class SAMAttention(nn.Module):+ def __init__(self, embedding_dim, num_heads, downsample_rate=1, kv_in_dim=None, device=None, dtype=None, operations=None):+ super().__init__()+ self.num_heads = num_heads+ internal_dim = embedding_dim // downsample_rate+ kv_dim = kv_in_dim if kv_in_dim is not None else embedding_dim+ self.q_proj = operations.Linear(embedding_dim, internal_dim, device=device, dtype=dtype)+ self.k_proj = operations.Linear(kv_dim, internal_dim, device=device, dtype=dtype)+ self.v_proj = operations.Linear(kv_dim, internal_dim, device=device, dtype=dtype)+ self.out_proj = operations.Linear(internal_dim, embedding_dim, device=device, dtype=dtype)++ def forward(self, q, k, v):+ q = self.q_proj(q)+ k = self.k_proj(k)+ v = self.v_proj(v)+ return self.out_proj(optimized_attention(q, k, v, self.num_heads))+++class TwoWayAttentionBlock(nn.Module):+ def __init__(self, embedding_dim, num_heads, mlp_dim=2048, attention_downsample_rate=2, skip_first_layer_pe=False, device=None, dtype=None, operations=None):+ super().__init__()+ self.skip_first_layer_pe = skip_first_layer_pe+ self.self_attn = SAMAttention(embedding_dim, num_heads, device=device, dtype=dtype, operations=operations)+ self.cross_attn_token_to_image = SAMAttention(embedding_dim, num_heads, downsample_rate=attention_downsample_rate, device=device, dtype=dtype, operations=operations)+ self.cross_attn_image_to_token = SAMAttention(embedding_dim, num_heads, downsample_rate=attention_downsample_rate, device=device, dtype=dtype, operations=operations)+ self.mlp = nn.Sequential(operations.Linear(embedding_dim, mlp_dim, device=device, dtype=dtype), nn.ReLU(), operations.Linear(mlp_dim, embedding_dim, device=device, dtype=dtype))+ self.norm1 = operations.LayerNorm(embedding_dim, device=device, dtype=dtype)+ self.norm2 = operations.LayerNorm(embedding_dim, device=device, dtype=dtype)+ self.norm3 = operations.LayerNorm(embedding_dim, device=device, dtype=dtype)+ self.norm4 = operations.LayerNorm(embedding_dim, device=device, dtype=dtype)++ def forward(self, queries, keys, query_pe, key_pe):+ if self.skip_first_layer_pe:+ queries = self.norm1(self.self_attn(queries, queries, queries))+ else:+ q = queries + query_pe+ queries = self.norm1(queries + self.self_attn(q, q, queries))+ q, k = queries + query_pe, keys + key_pe+ queries = self.norm2(queries + self.cross_attn_token_to_image(q, k, keys))+ queries = self.norm3(queries + self.mlp(queries))+ q, k = queries + query_pe, keys + key_pe+ keys = self.norm4(keys + self.cross_attn_image_to_token(k, q, queries))+ return queries, keys+++class TwoWayTransformer(nn.Module):+ def __init__(self, depth=2, embedding_dim=256, num_heads=8, mlp_dim=2048, attention_downsample_rate=2, device=None, dtype=None, operations=None):+ super().__init__()+ self.layers = nn.ModuleList([+ TwoWayAttentionBlock(embedding_dim, num_heads, mlp_dim, attention_downsample_rate,+ skip_first_layer_pe=(i == 0), device=device, dtype=dtype, operations=operations)+ for i in range(depth)+ ])+ self.final_attn_token_to_image = SAMAttention(embedding_dim, num_heads, downsample_rate=attention_downsample_rate, device=device, dtype=dtype, operations=operations)+ self.norm_final = operations.LayerNorm(embedding_dim, device=device, dtype=dtype)++ def forward(self, image_embedding, image_pe, point_embedding):+ queries, keys = point_embedding, image_embedding+ for layer in self.layers:+ queries, keys = layer(queries, keys, point_embedding, image_pe)+ q, k = queries + point_embedding, keys + image_pe+ queries = self.norm_final(queries + self.final_attn_token_to_image(q, k, keys))+ return queries, keys+++class PositionEmbeddingRandom(nn.Module):+ """Fourier feature positional encoding with random gaussian projection."""+ def __init__(self, num_pos_feats=64, scale=None):+ super().__init__()+ self.register_buffer("positional_encoding_gaussian_matrix", (scale or 1.0) * torch.randn(2, num_pos_feats))++ def _encode(self, normalized_coords):+ """Map normalized [0,1] coordinates to fourier features via random projection. Computes in fp32."""+ orig_dtype = normalized_coords.dtype+ proj_matrix = self.positional_encoding_gaussian_matrix.to(device=normalized_coords.device, dtype=torch.float32)+ projected = 2 * math.pi * (2 * normalized_coords.float() - 1) @ proj_matrix+ return torch.cat([projected.sin(), projected.cos()], dim=-1).to(orig_dtype)++ def forward(self, size, device=None):+ h, w = size+ dev = device if device is not None else self.positional_encoding_gaussian_matrix.device+ ones = torch.ones((h, w), device=dev, dtype=torch.float32)+ norm_xy = torch.stack([(ones.cumsum(1) - 0.5) / w, (ones.cumsum(0) - 0.5) / h], dim=-1)+ return self._encode(norm_xy).permute(2, 0, 1).unsqueeze(0)++ def forward_with_coords(self, pixel_coords, image_size):+ norm = pixel_coords.clone()+ norm[:, :, 0] /= image_size[1]+ norm[:, :, 1] /= image_size[0]+ return self._encode(norm)+++# ViTDet backbone + FPN neck++def window_partition(x: torch.Tensor, window_size: int):+ B, H, W, C = x.shape+ pad_h = (window_size - H % window_size) % window_size+ pad_w = (window_size - W % window_size) % window_size+ if pad_h > 0 or pad_w > 0:+ x = F.pad(x, (0, 0, 0, pad_w, 0, pad_h))+ Hp, Wp = H + pad_h, W + pad_w+ x = x.view(B, Hp // window_size, window_size, Wp // window_size, window_size, C)+ windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C)+ return windows, (Hp, Wp)+++def window_unpartition(windows: torch.Tensor, window_size: int, pad_hw, hw):+ Hp, Wp = pad_hw+ H, W = hw+ B = windows.shape[0] // (Hp * Wp // window_size // window_size)+ x = windows.view(B, Hp // window_size, Wp // window_size, window_size, window_size, -1)+ x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, Hp, Wp, -1)+ if Hp > H or Wp > W:+ x = x[:, :H, :W, :].contiguous()+ return x+++def rope_2d(end_x: int, end_y: int, dim: int, theta: float = 10000.0, scale_pos: float = 1.0):+ """Generate 2D axial RoPE using flux EmbedND. Returns [1, 1, HW, dim//2, 2, 2]."""+ t = torch.arange(end_x * end_y, dtype=torch.float32)+ ids = torch.stack([(t % end_x) * scale_pos,+ torch.div(t, end_x, rounding_mode="floor") * scale_pos], dim=-1)+ return EmbedND(dim=dim, theta=theta, axes_dim=[dim // 2, dim // 2])(ids.unsqueeze(0))+++class _ViTMLP(nn.Module):+ def __init__(self, dim, mlp_ratio=4.0, device=None, dtype=None, operations=None):+ super().__init__()+ hidden = int(dim * mlp_ratio)+ self.fc1 = operations.Linear(dim, hidden, device=device, dtype=dtype)+ self.act = nn.GELU()+ self.fc2 = operations.Linear(hidden, dim, device=device, dtype=dtype)++ def forward(self, x):+ return self.fc2(self.act(self.fc1(x)))+++class Attention(nn.Module):+ """ViTDet multi-head attention with fused QKV projection."""++ def __init__(self, dim, num_heads=8, qkv_bias=True, use_rope=False, device=None, dtype=None, operations=None):+ super().__init__()+ self.num_heads = num_heads+ self.head_dim = dim // num_heads+ self.use_rope = use_rope+ self.qkv = operations.Linear(dim, dim * 3, bias=qkv_bias, device=device, dtype=dtype)+ self.proj = operations.Linear(dim, dim, device=device, dtype=dtype)++ def forward(self, x, freqs_cis=None):+ B, N, C = x.shape+ qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, self.head_dim)+ q, k, v = qkv.permute(2, 0, 3, 1, 4).unbind(dim=0)+ if self.use_rope and freqs_cis is not None:+ q, k = apply_rope(q, k, freqs_cis)+ return self.proj(optimized_attention(q, k, v, self.num_heads, skip_reshape=True))+++class Block(nn.Module):+ def __init__(self, dim, num_heads, mlp_ratio=4.0, qkv_bias=True, window_size=0, use_rope=False, device=None, dtype=None, operations=None):+ super().__init__()+ self.window_size = window_size+ self.norm1 = operations.LayerNorm(dim, device=device, dtype=dtype)+ self.attn = Attention(dim, num_heads, qkv_bias, use_rope, device=device, dtype=dtype, operations=operations)+ self.norm2 = operations.LayerNorm(dim, device=device, dtype=dtype)+ self.mlp = _ViTMLP(dim, mlp_ratio, device=device, dtype=dtype, operations=operations)++ def forward(self, x, freqs_cis=None):+ shortcut = x+ x = self.norm1(x)+ if self.window_size > 0:+ H, W = x.shape[1], x.shape[2]+ x, pad_hw = window_partition(x, self.window_size)+ x = x.view(x.shape[0], self.window_size * self.window_size, -1)+ x = self.attn(x, freqs_cis=freqs_cis)+ x = x.view(-1, self.window_size, self.window_size, x.shape[-1])+ x = window_unpartition(x, self.window_size, pad_hw, (H, W))+ else:+ B, H, W, C = x.shape+ x = x.view(B, H * W, C)+ x = self.attn(x, freqs_cis=freqs_cis)+ x = x.view(B, H, W, C)+ x = shortcut + x+ x = x + self.mlp(self.norm2(x))+ return x+++class PatchEmbed(nn.Module):+ def __init__(self, patch_size=14, in_chans=3, embed_dim=1024, device=None, dtype=None, operations=None):+ super().__init__()+ self.proj = operations.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size, bias=False, device=device, dtype=dtype)++ def forward(self, x):+ return self.proj(x)+++class ViTDet(nn.Module):+ def __init__(self, img_size=1008, patch_size=14, embed_dim=1024, depth=32, num_heads=16, mlp_ratio=4.625, qkv_bias=True, window_size=24,+ global_att_blocks=(7, 15, 23, 31), use_rope=True, pretrain_img_size=336, device=None, dtype=None, operations=None, **kwargs):+ super().__init__()+ self.img_size = img_size+ self.patch_size = patch_size+ self.embed_dim = embed_dim+ self.num_heads = num_heads+ self.global_att_blocks = set(global_att_blocks)++ self.patch_embed = PatchEmbed(patch_size, 3, embed_dim, device=device, dtype=dtype, operations=operations)++ num_patches = (pretrain_img_size // patch_size) ** 2 + 1 # +1 for cls token+ self.pos_embed = nn.Parameter(torch.zeros(1, num_patches, embed_dim, device=device, dtype=dtype))++ self.ln_pre = operations.LayerNorm(embed_dim, device=device, dtype=dtype)++ grid_size = img_size // patch_size+ pretrain_grid = pretrain_img_size // patch_size++ self.blocks = nn.ModuleList()+ for i in range(depth):+ is_global = i in self.global_att_blocks+ self.blocks.append(Block(+ embed_dim, num_heads, mlp_ratio, qkv_bias,+ window_size=0 if is_global else window_size,+ use_rope=use_rope,+ device=device, dtype=dtype, operations=operations,+ ))++ if use_rope:+ rope_scale = pretrain_grid / grid_size+ self.register_buffer("freqs_cis", rope_2d(grid_size, grid_size, embed_dim // num_heads, scale_pos=rope_scale), persistent=False)+ self.register_buffer("freqs_cis_window", rope_2d(window_size, window_size, embed_dim // num_heads), persistent=False)+ else:+ self.freqs_cis = None+ self.freqs_cis_window = None++ def _get_pos_embed(self, num_tokens):+ pos = self.pos_embed+ if pos.shape[1] == num_tokens:+ return pos+ cls_pos = pos[:, :1]+ spatial_pos = pos[:, 1:]+ old_size = int(math.sqrt(spatial_pos.shape[1]))+ new_size = int(math.sqrt(num_tokens - 1)) if num_tokens > 1 else old_size+ spatial_2d = spatial_pos.reshape(1, old_size, old_size, -1).permute(0, 3, 1, 2)+ tiles_h = new_size // old_size + 1+ tiles_w = new_size // old_size + 1+ tiled = spatial_2d.tile([1, 1, tiles_h, tiles_w])[:, :, :new_size, :new_size]+ tiled = tiled.permute(0, 2, 3, 1).reshape(1, new_size * new_size, -1)+ return torch.cat([cls_pos, tiled], dim=1)++ def forward(self, x):+ x = self.patch_embed(x)+ B, C, Hp, Wp = x.shape+ x = x.permute(0, 2, 3, 1).reshape(B, Hp * Wp, C)++ pos = cast_to_input(self._get_pos_embed(Hp * Wp + 1), x)+ x = x + pos[:, 1:Hp * Wp + 1]++ x = x.view(B, Hp, Wp, C)+ x = self.ln_pre(x)++ freqs_cis_global = self.freqs_cis+ freqs_cis_win = self.freqs_cis_window+ if freqs_cis_global is not None:+ freqs_cis_global = cast_to_input(freqs_cis_global, x)+ if freqs_cis_win is not None:+ freqs_cis_win = cast_to_input(freqs_cis_win, x)++ for block in self.blocks:+ fc = freqs_cis_win if block.window_size > 0 else freqs_cis_global+ x = block(x, freqs_cis=fc)++ return x.permute(0, 3, 1, 2)+++class FPNScaleConv(nn.Module):+ def __init__(self, in_dim, out_dim, scale, device=None, dtype=None, operations=None):+ super().__init__()+ if scale == 4.0:+ self.dconv_2x2_0 = operations.ConvTranspose2d(in_dim, in_dim // 2, kernel_size=2, stride=2, device=device, dtype=dtype)+ self.dconv_2x2_1 = operations.ConvTranspose2d(in_dim // 2, in_dim // 4, kernel_size=2, stride=2, device=device, dtype=dtype)+ proj_in = in_dim // 4+ elif scale == 2.0:+ self.dconv_2x2 = operations.ConvTranspose2d(in_dim, in_dim // 2, kernel_size=2, stride=2, device=device, dtype=dtype)+ proj_in = in_dim // 2+ elif scale == 1.0:+ proj_in = in_dim+ elif scale == 0.5:+ self.pool = nn.MaxPool2d(kernel_size=2, stride=2)+ proj_in = in_dim+ self.scale = scale+ self.conv_1x1 = operations.Conv2d(proj_in, out_dim, kernel_size=1, device=device, dtype=dtype)+ self.conv_3x3 = operations.Conv2d(out_dim, out_dim, kernel_size=3, padding=1, device=device, dtype=dtype)++ def forward(self, x):+ if self.scale == 4.0:+ x = F.gelu(self.dconv_2x2_0(x))+ x = self.dconv_2x2_1(x)+ elif self.scale == 2.0:+ x = self.dconv_2x2(x)+ elif self.scale == 0.5:+ x = self.pool(x)+ x = self.conv_1x1(x)+ x = self.conv_3x3(x)+ return x+++class PositionEmbeddingSine(nn.Module):+ """2D sinusoidal position encoding (DETR-style) with result caching."""+ def __init__(self, num_pos_feats=256, temperature=10000.0, normalize=True, scale=None):+ super().__init__()+ assert num_pos_feats % 2 == 0+ self.half_dim = num_pos_feats // 2+ self.temperature = temperature+ self.normalize = normalize+ self.scale = scale if scale is not None else 2 * math.pi+ self._cache = {}++ def _sincos(self, vals):+ """Encode 1D values to interleaved sin/cos features."""+ freqs = self.temperature ** (2 * (torch.arange(self.half_dim, dtype=torch.float32, device=vals.device) // 2) / self.half_dim)+ raw = vals[..., None] * self.scale / freqs+ return torch.stack((raw[..., 0::2].sin(), raw[..., 1::2].cos()), dim=-1).flatten(-2)++ def _encode_xy(self, x, y):+ """Encode normalized x, y coordinates to sinusoidal features. Returns (pos_x, pos_y) each [N, half_dim]."""+ dim_t = self.temperature ** (2 * (torch.arange(self.half_dim, dtype=torch.float32, device=x.device) // 2) / self.half_dim)+ pos_x = x[:, None] * self.scale / dim_t+ pos_y = y[:, None] * self.scale / dim_t+ pos_x = torch.stack((pos_x[:, 0::2].sin(), pos_x[:, 1::2].cos()), dim=2).flatten(1)+ pos_y = torch.stack((pos_y[:, 0::2].sin(), pos_y[:, 1::2].cos()), dim=2).flatten(1)+ return pos_x, pos_y++ def encode_boxes(self, cx, cy, w, h):+ """Encode box center + size to [N, d_model+2] features."""+ pos_x, pos_y = self._encode_xy(cx, cy)+ return torch.cat((pos_y, pos_x, h[:, None], w[:, None]), dim=1)++ def forward(self, x):+ B, C, H, W = x.shape+ key = (H, W, x.device)+ if key not in self._cache:+ gy = torch.arange(H, dtype=torch.float32, device=x.device)+ gx = torch.arange(W, dtype=torch.float32, device=x.device)+ if self.normalize:+ gy, gx = gy / (H - 1 + 1e-6), gx / (W - 1 + 1e-6)+ yy, xx = torch.meshgrid(gy, gx, indexing="ij")+ self._cache[key] = torch.cat((self._sincos(yy), self._sincos(xx)), dim=-1).permute(2, 0, 1).unsqueeze(0)+ return self._cache[key].expand(B, -1, -1, -1)+++class SAM3VisionBackbone(nn.Module):+ def __init__(self, embed_dim=1024, d_model=256, multiplex=False, device=None, dtype=None, operations=None, **kwargs):+ super().__init__()+ self.trunk = ViTDet(embed_dim=embed_dim, device=device, dtype=dtype, operations=operations, **kwargs)+ self.position_encoding = PositionEmbeddingSine(num_pos_feats=d_model, normalize=True)+ self.multiplex = multiplex++ fpn_args = dict(device=device, dtype=dtype, operations=operations)+ if multiplex:+ scales = [4.0, 2.0, 1.0]+ self.convs = nn.ModuleList([FPNScaleConv(embed_dim, d_model, s, **fpn_args) for s in scales])+ self.propagation_convs = nn.ModuleList([FPNScaleConv(embed_dim, d_model, s, **fpn_args) for s in scales])+ self.interactive_convs = nn.ModuleList([FPNScaleConv(embed_dim, d_model, s, **fpn_args) for s in scales])+ else:+ scales = [4.0, 2.0, 1.0, 0.5]+ self.convs = nn.ModuleList([FPNScaleConv(embed_dim, d_model, s, **fpn_args) for s in scales])+ self.sam2_convs = nn.ModuleList([FPNScaleConv(embed_dim, d_model, s, **fpn_args) for s in scales])++ def forward(self, images, need_tracker=False, tracker_mode=None, cached_trunk=None, tracker_only=False):+ backbone_out = cached_trunk if cached_trunk is not None else self.trunk(images)++ if tracker_only:+ # Skip detector FPN when only tracker features are needed (video tracking)+ if self.multiplex:+ tracker_convs = self.propagation_convs if tracker_mode == "propagation" else self.interactive_convs+ else:+ tracker_convs = self.sam2_convs+ tracker_features = [conv(backbone_out) for conv in tracker_convs]+ tracker_positions = [cast_to_input(self.position_encoding(f), f) for f in tracker_features]+ return None, None, tracker_features, tracker_positions++ features = [conv(backbone_out) for conv in self.convs]+ positions = [cast_to_input(self.position_encoding(f), f) for f in features]++ if self.multiplex:+ if tracker_mode == "propagation":+ tracker_convs = self.propagation_convs+ elif tracker_mode == "interactive":+ tracker_convs = self.interactive_convs+ else:+ return features, positions, None, None+ elif need_tracker:+ tracker_convs = self.sam2_convs+ else:+ return features, positions, None, None++ tracker_features = [conv(backbone_out) for conv in tracker_convs]+ tracker_positions = [cast_to_input(self.position_encoding(f), f) for f in tracker_features]+ return features, positions, tracker_features, tracker_positionsdiff --git a/comfy/ldm/sam3/tracker.py b/comfy/ldm/sam3/tracker.pynew file mode 100644index 000000000..6ff6369d1--- /dev/null+++ b/comfy/ldm/sam3/tracker.py@@ -0,0 +1,1785 @@+# SAM3 video tracker: memory encoder, memory attention, SAM mask decoder/prompt encoder.++import numpy as np+import torch+import torch.nn as nn+import torch.nn.functional as F+from tqdm import tqdm++try:+ import cv2+ _HAS_CV2 = True+except ImportError:+ from scipy import ndimage+ _HAS_CV2 = False++import comfy.model_management+from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.sam3.sam import rope_2d, PositionEmbeddingSine+from comfy.ops import cast_to_input+from comfy.ldm.flux.math import apply_rope1+from comfy.ldm.cascade.common import LayerNorm2d_op+from comfy.ldm.sam3.sam import MLP, PositionEmbeddingRandom+from comfy.ldm.sam3.sam import TwoWayTransformer as SAMTwoWayTransformer++NO_OBJ_SCORE = -1024.0+++def to_spatial(x, H, W):+ """Reshape (B, H*W, C) → (B, C, H, W)."""+ return x.view(x.shape[0], H, W, -1).permute(0, 3, 1, 2)++class MultiplexState:+ """Tracks object-to-slot assignments for multiplex tracking. Provides mux/demux operations."""++ def __init__(self, num_objects, multiplex_count, device, dtype):+ self.multiplex_count = multiplex_count+ self.device = device+ self.dtype = dtype+ self._build(num_objects)++ def mux(self, x):+ """[N_obj, ...] -> [num_buckets, multiplex_count, ...]"""+ out_shape = (self.num_buckets, self.multiplex_count) + x.shape[1:]+ return (self.mux_matrix.to(device=x.device, dtype=x.dtype) @ x.reshape(self.total_valid_entries, -1)).view(out_shape)++ def demux(self, x):+ """[num_buckets, multiplex_count, ...] -> [N_obj, ...]"""+ out_shape = (self.total_valid_entries,) + x.shape[2:]+ flat = x.reshape(self.num_buckets * self.multiplex_count, -1)+ return (self.demux_matrix.to(device=x.device, dtype=x.dtype) @ flat).view(out_shape)++ def get_valid_object_mask(self):+ """[num_buckets, multiplex_count] bool tensor, True for valid slots."""+ return (self.mux_matrix.sum(dim=1) > 0).reshape(self.num_buckets, self.multiplex_count)++ def _build(self, num_objects):+ M = self.multiplex_count+ self.num_buckets = (num_objects + M - 1) // M+ self.total_valid_entries = num_objects+ total_slots = self.num_buckets * M+ self.mux_matrix = torch.zeros(total_slots, num_objects, device=self.device, dtype=self.dtype)+ self.demux_matrix = torch.zeros(num_objects, total_slots, device=self.device, dtype=self.dtype)+ oids = torch.arange(num_objects, device=self.device)+ slots = (oids // M) * M + (oids % M)+ self.mux_matrix[slots, oids] = 1.0+ self.demux_matrix[oids, slots] = 1.0++ def add_objects(self, n_new):+ """Grow multiplex state for n_new additional objects."""+ self._build(self.total_valid_entries + n_new)++def _compute_mask_overlap(masks_a, masks_b):+ """Max of IoU and IoM (intersection over minimum area). More robust to size differences."""+ a_flat = (masks_a > 0).float().flatten(1)+ b_flat = (masks_b > 0).float().flatten(1)+ intersection = a_flat @ b_flat.T+ area_a = a_flat.sum(1, keepdim=True)+ area_b = b_flat.sum(1, keepdim=True).T+ iou = intersection / (area_a + area_b - intersection).clamp(min=1)+ iom = intersection / torch.min(area_a.expand_as(iou), area_b.expand_as(iou)).clamp(min=1)+ return torch.max(iou, iom)+++def _nms_masks(masks, scores, thresh=0.5):+ """Mask-based NMS using IoU+IoM overlap. Returns (filtered_masks, filtered_scores)."""+ order = scores.argsort(descending=True)+ masks, scores = masks[order], scores[order]+ keep = []+ for i in range(masks.shape[0]):+ if keep:+ if _compute_mask_overlap(masks[i:i+1], masks[torch.tensor(keep, device=masks.device)]).max() >= thresh:+ continue+ keep.append(i)+ return masks[keep], scores[keep]+++def _get_connected_components(mask_bin):+ """Get connected component labels and areas. mask_bin: [B, 1, H, W] uint8."""+ labels_list, areas_list = [], []+ for i in range(mask_bin.shape[0]):+ m = mask_bin[i, 0].cpu().numpy()+ if _HAS_CV2:+ _, labeled, stats, _ = cv2.connectedComponentsWithStats(m, connectivity=8)+ areas = stats[labeled, cv2.CC_STAT_AREA].astype('int32')+ else:+ labeled, num_features = ndimage.label(m)+ areas = np.zeros_like(m, dtype=np.int32)+ for c in range(1, num_features + 1):+ component = labeled == c+ areas[component] = component.sum()+ labels_list.append(torch.from_numpy(labeled).to(mask_bin.device))+ areas_list.append(torch.from_numpy(areas).to(device=mask_bin.device, dtype=torch.int32))+ return torch.stack(labels_list).unsqueeze(1), torch.stack(areas_list).unsqueeze(1)+++def fill_holes_in_mask_scores(mask, max_area=0):+ """Remove small foreground sprinkles and fill small background holes using connected components."""+ if max_area <= 0:+ return mask++ # Fill holes: small connected components in background → foreground+ mask_bg = (mask <= 0).to(torch.uint8)+ _, areas_bg = _get_connected_components(mask_bg)+ small_bg = mask_bg.bool() & (areas_bg <= max_area)+ mask = torch.where(small_bg, 0.1, mask)++ # Remove sprinkles: small connected components in foreground → background+ # Only remove if area < min(max_area, half of total foreground area)+ mask_fg = (mask > 0).to(torch.uint8)+ fg_area_thresh = mask_fg.sum(dim=(2, 3), keepdim=True, dtype=torch.int32)+ fg_area_thresh.floor_divide_(2).clamp_(max=max_area)+ _, areas_fg = _get_connected_components(mask_fg)+ small_fg = mask_fg.bool() & (areas_fg <= fg_area_thresh)+ mask = torch.where(small_fg, -0.1, mask)++ return mask+++def apply_rope_memory(q, k, freqs, num_heads, num_k_exclude_rope=0):+ """Apply 2D axial RoPE to memory attention using flux rope format.++ Args:+ q: [B, Nq, C] projected queries (current frame features)+ k: [B, Nk, C] projected keys (memory tokens)+ freqs: [1, Nq, dim//2, 2, 2] flux-format rotation matrices for one frame+ num_heads: number of attention heads+ num_k_exclude_rope: number of trailing k tokens to skip RoPE (object pointers)+ """+ B, Nq, C = q.shape+ head_dim = C // num_heads++ # freqs shape: [1, 1, Nq, dim//2, 2, 2] (heads broadcast dim already included)+ q_h = q.view(B, Nq, num_heads, head_dim).transpose(1, 2)+ q_h = apply_rope1(q_h, freqs)+ q = q_h.transpose(1, 2).reshape(B, Nq, C)++ # Apply RoPE to k (excluding last num_k_exclude_rope tokens)+ Nk = k.shape[1]+ num_k_rope = Nk - num_k_exclude_rope+ if num_k_rope > 0:+ # Repeat freqs for multiple frames of spatial memory+ Nf = freqs.shape[2] # spatial positions in one frame+ if num_k_rope > Nf:+ r = (num_k_rope + Nf - 1) // Nf+ pe_k = freqs.repeat(1, 1, r, 1, 1, 1)[:, :, :num_k_rope]+ else:+ pe_k = freqs[:, :, :num_k_rope]++ k_h = k[:, :num_k_rope].view(B, num_k_rope, num_heads, head_dim).transpose(1, 2)+ k_h = apply_rope1(k_h, pe_k)+ k = k.clone()+ k[:, :num_k_rope] = k_h.transpose(1, 2).reshape(B, num_k_rope, C)++ return q, k+++def get_1d_sine_pe(pos_inds, dim, temperature=10000):+ """1D sinusoidal positional encoding for temporal positions."""+ pe_dim = dim // 2+ dim_t = torch.arange(pe_dim, dtype=torch.float32, device=pos_inds.device)+ dim_t = temperature ** (2 * (dim_t // 2) / pe_dim)+ pos_embed = pos_inds.unsqueeze(-1) / dim_t+ return torch.cat([pos_embed.sin(), pos_embed.cos()], dim=-1)+++def _pad_to_buckets(tensor, target_buckets):+ """Pad a [num_buckets, ...] tensor to target_buckets along dim 0 if needed."""+ if tensor.shape[0] >= target_buckets:+ return tensor+ pad_shape = (target_buckets - tensor.shape[0],) + tensor.shape[1:]+ return torch.cat([tensor, torch.zeros(pad_shape, device=tensor.device, dtype=tensor.dtype)], dim=0)+++def pack_masks(masks):+ """Pack binary masks [*, H, W] to bit-packed [*, H, W//8] uint8. W must be divisible by 8."""+ binary = masks > 0+ shifts = torch.arange(8, device=masks.device)+ return (binary.view(*masks.shape[:-1], -1, 8) * (1 << shifts)).sum(-1).byte()+++def unpack_masks(packed):+ """Unpack bit-packed [*, H, W//8] uint8 to bool [*, H, W*8]."""+ shifts = torch.arange(8, device=packed.device)+ return ((packed.unsqueeze(-1) >> shifts) & 1).view(*packed.shape[:-1], -1).bool()+++def _compute_backbone(backbone_fn, frame, frame_idx=None):+ """Compute backbone features for a single frame. Returns (vision_feats, vision_pos, feat_sizes, features, trunk_out)."""+ features, positions, trunk_out = backbone_fn(frame, frame_idx=frame_idx)+ feat_sizes = [(x.shape[-2], x.shape[-1]) for x in features]+ vision_feats = [x.flatten(2).permute(0, 2, 1) for x in features]+ vision_pos = [x.flatten(2).permute(0, 2, 1) for x in positions]+ return vision_feats, vision_pos, feat_sizes, features, trunk_out+++def collect_memory_tokens(output_dict, frame_idx, num_maskmem, maskmem_tpos_enc, device,+ collect_image_feats=False, tpos_v2=False, num_buckets=None):+ """Collect spatial memory, position encodings, and optionally image features from past frames."""+ to_cat_memory, to_cat_memory_pos = [], []+ to_cat_image_feat, to_cat_image_pos = [], []++ def _append(out, tpos_idx):+ feats = out["maskmem_features"].to(device)+ if num_buckets is not None:+ feats = _pad_to_buckets(feats, num_buckets)+ to_cat_memory.append(feats.flatten(2).permute(0, 2, 1))+ enc = out["maskmem_pos_enc"][-1].to(device).flatten(2).permute(0, 2, 1)+ if num_buckets is not None:+ enc = _pad_to_buckets(enc, num_buckets)+ tpos = cast_to_input(maskmem_tpos_enc[tpos_idx], enc)+ to_cat_memory_pos.append(enc + tpos)+ if collect_image_feats and "image_features" in out:+ to_cat_image_feat.append(out["image_features"].to(device))+ to_cat_image_pos.append(out["image_pos_enc"].to(device) + tpos)++ cond_outputs = output_dict["cond_frame_outputs"]+ for t, out in cond_outputs.items():+ if tpos_v2:+ t_pos = frame_idx - t+ tpos_idx = num_maskmem - t_pos - 1 if 0 < t_pos < num_maskmem else num_maskmem - 1+ else:+ tpos_idx = num_maskmem - 1+ _append(out, tpos_idx)++ for t_pos in range(1, num_maskmem):+ out = output_dict["non_cond_frame_outputs"].get(frame_idx - (num_maskmem - t_pos), None)+ if out is None or out.get("maskmem_features") is None:+ continue+ _append(out, num_maskmem - t_pos - 1)++ return to_cat_memory, to_cat_memory_pos, to_cat_image_feat, to_cat_image_pos, cond_outputs+++def compute_tpos_enc(rel_pos_list, device, d_model, proj_layer, dtype=None, max_abs_pos=None):+ """Temporal position encoding for object pointers."""+ pos_enc = torch.tensor(rel_pos_list, dtype=torch.float32, device=device) / max((max_abs_pos or 2) - 1, 1)+ pos_enc = get_1d_sine_pe(pos_enc, dim=d_model)+ if dtype is not None:+ pos_enc = pos_enc.to(dtype)+ return proj_layer(pos_enc)+++def forward_sam_heads(backbone_features, prompt_encoder, mask_decoder, obj_ptr_proj, no_obj_fn,+ image_size, point_inputs=None, mask_inputs=None, box_inputs=None,+ high_res_features=None, multimask_output=False):+ """Shared SAM prompt encoder + mask decoder forward for both SAM3 and SAM3.1 trackers."""+ device = backbone_features.device+ # Batch size from inputs (mask_inputs may have N_obj > 1 while backbone is batch 1)+ if mask_inputs is not None:+ B = mask_inputs.shape[0]+ elif box_inputs is not None:+ B = box_inputs.shape[0]+ elif point_inputs is not None:+ B = point_inputs["point_coords"].shape[0]+ else:+ B = backbone_features.shape[0]++ if point_inputs is not None:+ sam_point_coords = point_inputs["point_coords"]+ sam_point_labels = point_inputs["point_labels"]+ else:+ sam_point_coords = torch.zeros(B, 1, 2, device=device)+ sam_point_labels = -torch.ones(B, 1, dtype=torch.int32, device=device)++ if mask_inputs is not None:+ prompt_size = (prompt_encoder.image_embedding_size[0] * 4, prompt_encoder.image_embedding_size[1] * 4)+ if mask_inputs.shape[-2:] != prompt_size:+ sam_mask_prompt = F.interpolate(mask_inputs, size=prompt_size, mode="bilinear", align_corners=False, antialias=True)+ else:+ sam_mask_prompt = mask_inputs+ else:+ sam_mask_prompt = None++ sparse, dense = prompt_encoder(points=(sam_point_coords, sam_point_labels), boxes=box_inputs, masks=sam_mask_prompt)+ sparse = cast_to_input(sparse, backbone_features)+ dense = cast_to_input(dense, backbone_features)+ image_pe = cast_to_input(prompt_encoder.get_dense_pe(), backbone_features)++ low_res_multimasks, ious, sam_output_tokens, object_score_logits = mask_decoder(+ image_embeddings=backbone_features, image_pe=image_pe,+ sparse_prompt_embeddings=sparse, dense_prompt_embeddings=dense,+ high_res_features=high_res_features, multimask_output=multimask_output, return_all=True,+ )++ is_obj_appearing = object_score_logits > 0+ low_res_multimasks = torch.where(is_obj_appearing[:, None, None], low_res_multimasks,+ torch.tensor(NO_OBJ_SCORE, device=device, dtype=low_res_multimasks.dtype))+ high_res_multimasks = F.interpolate(low_res_multimasks, size=(image_size, image_size), mode="bilinear", align_corners=False)++ sam_output_token = sam_output_tokens[:, 0]+ if multimask_output:+ best_iou_inds = torch.argmax(ious, dim=-1)+ batch_inds = torch.arange(B, device=device)+ low_res_masks = low_res_multimasks[batch_inds, best_iou_inds].unsqueeze(1)+ high_res_masks = high_res_multimasks[batch_inds, best_iou_inds].unsqueeze(1)+ if sam_output_tokens.size(1) > 1:+ sam_output_token = sam_output_tokens[batch_inds, best_iou_inds]+ else:+ low_res_masks, high_res_masks = low_res_multimasks, high_res_multimasks++ obj_ptr = obj_ptr_proj(sam_output_token)+ obj_ptr = no_obj_fn(obj_ptr, is_obj_appearing)++ return low_res_masks, high_res_masks, obj_ptr, object_score_logits+++def use_mask_as_output(backbone_features, high_res_features, mask_inputs, mask_downsample,+ prompt_encoder, mask_decoder, obj_ptr_proj, no_obj_fn, image_size, backbone_stride):+ """Shared mask-as-output for both SAM3 and SAM3.1 trackers."""+ out_scale, out_bias = 20.0, -10.0+ mask_inputs_float = cast_to_input(mask_inputs, backbone_features)+ high_res_masks = mask_inputs_float * out_scale + out_bias+ low_res_masks = F.interpolate(high_res_masks, size=(image_size // backbone_stride * 4,) * 2,+ mode="bilinear", align_corners=False, antialias=True)+ _, _, obj_ptr, _ = forward_sam_heads(+ backbone_features, prompt_encoder, mask_decoder, obj_ptr_proj, no_obj_fn,+ image_size, mask_inputs=mask_downsample(mask_inputs_float), high_res_features=high_res_features,+ )+ is_obj_appearing = torch.any(mask_inputs.flatten(1) > 0.0, dim=1)[..., None]+ alpha = is_obj_appearing.to(obj_ptr.dtype)+ object_score_logits = out_scale * alpha + out_bias+ return low_res_masks, high_res_masks, obj_ptr, object_score_logits+++# Split attention with configurable input dims (for asymmetric cross-attention)+class SplitAttn(nn.Module):+ def __init__(self, embed_dim, num_heads=1, kv_dim=None, internal_dim=None, device=None, dtype=None, operations=None):+ super().__init__()+ self.num_heads = num_heads+ kv_dim = kv_dim or embed_dim+ internal_dim = internal_dim or embed_dim+ self.q_proj = operations.Linear(embed_dim, internal_dim, device=device, dtype=dtype)+ self.k_proj = operations.Linear(kv_dim, internal_dim, device=device, dtype=dtype)+ self.v_proj = operations.Linear(kv_dim, internal_dim, device=device, dtype=dtype)+ self.out_proj = operations.Linear(internal_dim, embed_dim, device=device, dtype=dtype)++ def forward(self, q, k=None, v=None, rope=None, num_k_exclude_rope=0):+ if k is None:+ k = q+ if v is None:+ v = k+ q = self.q_proj(q)+ k = self.k_proj(k)+ v = self.v_proj(v)+ if rope is not None:+ q, k = apply_rope_memory(q, k, rope, self.num_heads, num_k_exclude_rope)+ out = optimized_attention(q, k, v, self.num_heads)+ return self.out_proj(out)+++class MemoryAttnLayer(nn.Module):+ def __init__(self, d_model=256, num_heads=1, kv_dim=64, dim_ff=2048, device=None, dtype=None, operations=None):+ super().__init__()+ self.num_heads = num_heads+ self.self_attn = SplitAttn(d_model, num_heads, device=device, dtype=dtype, operations=operations)+ self.cross_attn_image = SplitAttn(d_model, num_heads, kv_dim=kv_dim, device=device, dtype=dtype, operations=operations)+ self.linear1 = operations.Linear(d_model, dim_ff, device=device, dtype=dtype)+ self.linear2 = operations.Linear(dim_ff, d_model, device=device, dtype=dtype)+ self.norm1 = operations.LayerNorm(d_model, device=device, dtype=dtype)+ self.norm2 = operations.LayerNorm(d_model, device=device, dtype=dtype)+ self.norm3 = operations.LayerNorm(d_model, device=device, dtype=dtype)++ def forward(self, x, memory, memory_pos=None, rope=None, num_k_exclude_rope=0):+ x = x + self.self_attn(self.norm1(x), rope=rope)+ mem_k = memory + memory_pos if memory_pos is not None else memory+ x = x + self.cross_attn_image(self.norm2(x), mem_k, memory, rope=rope, num_k_exclude_rope=num_k_exclude_rope)+ normed = self.norm3(x)+ x = x + self.linear2(F.relu(self.linear1(normed)))+ return x+++class MemoryAttnEncoder(nn.Module):+ def __init__(self, d_model=256, num_heads=1, kv_dim=64, dim_ff=2048, num_layers=4, image_size=1008, patch_size=14,+ device=None, dtype=None, operations=None):+ super().__init__()+ self.layers = nn.ModuleList([+ MemoryAttnLayer(d_model, num_heads, kv_dim, dim_ff, device=device, dtype=dtype, operations=operations)+ for _ in range(num_layers)+ ])+ self.norm = operations.LayerNorm(d_model, device=device, dtype=dtype)+ hw = image_size // patch_size+ self.register_buffer("_rope", rope_2d(hw, hw, d_model // num_heads), persistent=False)++ def forward(self, x, memory, src_pos=None, memory_pos=None, num_k_exclude_rope=0):+ if src_pos is not None:+ x = x + 0.1 * src_pos++ rope = self._rope.to(device=x.device)+ for layer in self.layers:+ x = layer(x, memory, memory_pos=memory_pos, rope=rope, num_k_exclude_rope=num_k_exclude_rope)+ return self.norm(x)+++class MemoryTransformer(nn.Module):+ def __init__(self, d_model=256, num_heads=1, kv_dim=64, dim_ff=2048, num_layers=4, device=None, dtype=None, operations=None):+ super().__init__()+ self.encoder = MemoryAttnEncoder(d_model, num_heads, kv_dim, dim_ff, num_layers, device=device, dtype=dtype, operations=operations)+++def _upscale_masks(output_upscaling, conv_s0, conv_s1, src_out, high_res_features):+ """Shared upscaling for SAM mask decoders: deconv + high-res feature integration."""+ dc1, ln1, act1, dc2, act2 = output_upscaling+ if high_res_features is not None:+ upscaled = act1(ln1(dc1(src_out) + conv_s1(high_res_features[1])))+ upscaled = act2(dc2(upscaled) + conv_s0(high_res_features[0]))+ else:+ upscaled = act2(dc2(act1(ln1(dc1(src_out)))))+ return upscaled+++class SAMMaskDecoder(nn.Module):+ def __init__(self, d_model=256, num_multimask_outputs=3, device=None, dtype=None, operations=None):+ super().__init__()+ self.num_mask_tokens = num_multimask_outputs + 1++ self.transformer = SAMTwoWayTransformer(depth=2, embedding_dim=d_model, num_heads=8, mlp_dim=2048, device=device, dtype=dtype, operations=operations)++ self.iou_token = operations.Embedding(1, d_model, device=device, dtype=dtype)+ self.mask_tokens = operations.Embedding(self.num_mask_tokens, d_model, device=device, dtype=dtype)+ self.obj_score_token = operations.Embedding(1, d_model, device=device, dtype=dtype)++ # Output upscaling: d_model -> d_model//4 -> d_model//8 at 4x resolution+ LN2d = LayerNorm2d_op(operations)+ self.output_upscaling = nn.Sequential(+ operations.ConvTranspose2d(d_model, d_model // 4, kernel_size=2, stride=2, device=device, dtype=dtype), LN2d(d_model // 4, device=device, dtype=dtype), nn.GELU(),+ operations.ConvTranspose2d(d_model // 4, d_model // 8, kernel_size=2, stride=2, device=device, dtype=dtype), nn.GELU(),+ )++ # High-res feature integration+ self.conv_s0 = operations.Conv2d(d_model, d_model // 8, kernel_size=1, device=device, dtype=dtype)+ self.conv_s1 = operations.Conv2d(d_model, d_model // 4, kernel_size=1, device=device, dtype=dtype)++ # Per-mask hypernetwork MLPs+ self.output_hypernetworks_mlps = nn.ModuleList([+ MLP(d_model, d_model, d_model // 8, 3, device=device, dtype=dtype, operations=operations)+ for _ in range(self.num_mask_tokens)+ ])++ self.iou_prediction_head = MLP(d_model, d_model, self.num_mask_tokens, 3, device=device, dtype=dtype, operations=operations)+ self.pred_obj_score_head = MLP(d_model, d_model, 1, 3, device=device, dtype=dtype, operations=operations)++ def forward(self, image_embeddings, image_pe, sparse_prompt_embeddings, dense_prompt_embeddings,+ high_res_features=None, multimask_output=False, return_all=False):+ B = sparse_prompt_embeddings.shape[0]+ ref = sparse_prompt_embeddings+ # Token order: [obj_score(1), iou(1), mask(num_mask_tokens)]+ tokens = torch.cat([cast_to_input(self.obj_score_token.weight, ref),+ cast_to_input(self.iou_token.weight, ref),+ cast_to_input(self.mask_tokens.weight, ref)], dim=0)+ tokens = torch.cat([tokens.unsqueeze(0).expand(B, -1, -1), sparse_prompt_embeddings], dim=1)++ src = image_embeddings+ if src.shape[0] != B:+ src = src.expand(B, -1, -1, -1)+ src = src + dense_prompt_embeddings+ pos_src = image_pe.expand(B, -1, -1, -1)++ b, c, h, w = src.shape+ src_flat = src.flatten(2).permute(0, 2, 1)+ pos_flat = pos_src.flatten(2).permute(0, 2, 1)++ hs, src_out = self.transformer(src_flat, pos_flat, tokens)++ obj_score_token_out = hs[:, 0, :]+ iou_token_out = hs[:, 1, :]+ mask_tokens_out = hs[:, 2:2 + self.num_mask_tokens, :]++ src_out = src_out.permute(0, 2, 1).view(b, c, h, w)+ upscaled = _upscale_masks(self.output_upscaling, self.conv_s0, self.conv_s1, src_out, high_res_features)++ hyper_in = torch.stack([+ mlp(mask_tokens_out[:, i, :]) for i, mlp in enumerate(self.output_hypernetworks_mlps)+ ], dim=1)++ masks = (hyper_in @ upscaled.flatten(2)).view(B, self.num_mask_tokens, upscaled.shape[2], upscaled.shape[3])+ iou_pred = self.iou_prediction_head(iou_token_out)+ object_score_logits = self.pred_obj_score_head(obj_score_token_out)++ if multimask_output:+ out_masks = masks[:, 1:]+ out_iou = iou_pred[:, 1:]+ out_tokens = mask_tokens_out[:, 1:]+ else:+ out_masks = masks[:, 0:1]+ out_iou = iou_pred[:, 0:1]+ out_tokens = mask_tokens_out[:, 0:1]++ if return_all:+ return out_masks, out_iou, out_tokens, object_score_logits+ return out_masks, out_iou+++class SAMPromptEncoder(nn.Module):+ def __init__(self, d_model=256, image_embedding_size=(72, 72), input_image_size=(1008, 1008), device=None, dtype=None, operations=None):+ super().__init__()+ self.embed_dim = d_model+ self.image_embedding_size = image_embedding_size+ self.input_image_size = input_image_size++ self.pe_layer = PositionEmbeddingRandom(d_model // 2)+ self.point_embeddings = nn.ModuleList([+ operations.Embedding(1, d_model, device=device, dtype=dtype) for _ in range(4)+ ])+ self.not_a_point_embed = operations.Embedding(1, d_model, device=device, dtype=dtype)++ LN2d = LayerNorm2d_op(operations)+ self.mask_downscaling = nn.Sequential(+ operations.Conv2d(1, 4, kernel_size=2, stride=2, device=device, dtype=dtype),+ LN2d(4, device=device, dtype=dtype), nn.GELU(),+ operations.Conv2d(4, 16, kernel_size=2, stride=2, device=device, dtype=dtype),+ LN2d(16, device=device, dtype=dtype), nn.GELU(),+ operations.Conv2d(16, d_model, kernel_size=1, device=device, dtype=dtype),+ )+ self.no_mask_embed = operations.Embedding(1, d_model, device=device, dtype=dtype)++ def get_dense_pe(self):+ return self.pe_layer(self.image_embedding_size)++ def forward(self, points=None, boxes=None, masks=None):+ ref = points[0] if points is not None else boxes if boxes is not None else masks+ B = 1+ sparse = torch.empty((B, 0, self.embed_dim), device=ref.device, dtype=ref.dtype)++ if points is not None:+ coords, labels = points+ B = coords.shape[0]+ # Pad with an extra point (label=-1) when no boxes are provided (matching reference)+ if boxes is None:+ coords = torch.cat([coords, torch.zeros(B, 1, 2, device=coords.device, dtype=coords.dtype)], dim=1)+ labels = torch.cat([labels, -torch.ones(B, 1, device=labels.device, dtype=labels.dtype)], dim=1)+ pe = self.pe_layer.forward_with_coords(coords + 0.5, self.input_image_size)+ for i in range(4):+ pe[labels == i] += cast_to_input(self.point_embeddings[i].weight, ref)+ invalid = (labels == -1)+ pe[invalid] = 0.0+ pe[invalid] += cast_to_input(self.not_a_point_embed.weight, ref)+ sparse = torch.cat([sparse.expand(B, -1, -1), pe], dim=1)++ if boxes is not None:+ B = boxes.shape[0]+ corners = self.pe_layer.forward_with_coords((boxes.reshape(-1, 2, 2) + 0.5), self.input_image_size)+ corners[:, 0] += cast_to_input(self.point_embeddings[2].weight, ref)+ corners[:, 1] += cast_to_input(self.point_embeddings[3].weight, ref)+ sparse = torch.cat([sparse.expand(B, -1, -1), corners], dim=1)++ if masks is not None:+ dense = self.mask_downscaling(masks)+ else:+ dense = cast_to_input(self.no_mask_embed.weight, ref).reshape(1, -1, 1, 1).expand(+ B, -1, self.image_embedding_size[0], self.image_embedding_size[1])++ return sparse, dense+++class CXBlock(nn.Module):+ def __init__(self, dim=256, kernel_size=7, device=None, dtype=None, operations=None):+ super().__init__()+ self.dwconv = operations.Conv2d(dim, dim, kernel_size=kernel_size, padding=kernel_size // 2, groups=dim, device=device, dtype=dtype)+ self.norm = operations.LayerNorm(dim, device=device, dtype=dtype)+ self.pwconv1 = operations.Linear(dim, 4 * dim, device=device, dtype=dtype)+ self.pwconv2 = operations.Linear(4 * dim, dim, device=device, dtype=dtype)+ self.gamma = nn.Parameter(torch.ones(dim, device=device, dtype=dtype))++ def forward(self, x):+ residual = x+ x = self.dwconv(x).permute(0, 2, 3, 1)+ x = self.pwconv2(F.gelu(self.pwconv1(self.norm(x))))+ x.mul_(cast_to_input(self.gamma, x))+ return residual + x.permute(0, 3, 1, 2)+++class MaskDownSampler(nn.Module):+ def __init__(self, out_dim=256, in_chans=1, channels=None, interpol_size=(1152, 1152), device=None, dtype=None, operations=None):+ super().__init__()+ self.interpol_size = list(interpol_size) if interpol_size else None+ if channels is None:+ channels = [4, 16, 64, out_dim] # SAM3 default+ LN2d = LayerNorm2d_op(operations)+ layers = []+ prev = in_chans+ for ch in channels:+ layers += [operations.Conv2d(prev, ch, kernel_size=3, stride=2, padding=1, device=device, dtype=dtype),+ LN2d(ch, device=device, dtype=dtype), nn.GELU()]+ prev = ch+ layers.append(operations.Conv2d(prev, out_dim, kernel_size=1, device=device, dtype=dtype))+ self.encoder = nn.Sequential(*layers)++ def forward(self, x):+ if self.interpol_size is not None and list(x.shape[-2:]) != self.interpol_size:+ x = F.interpolate(x, size=self.interpol_size, mode="bilinear", align_corners=False, antialias=True)+ return self.encoder(x)+++class Fuser(nn.Module):+ def __init__(self, dim=256, num_layers=2, device=None, dtype=None, operations=None):+ super().__init__()+ self.layers = nn.Sequential(*[CXBlock(dim, device=device, dtype=dtype, operations=operations) for _ in range(num_layers)])++ def forward(self, x):+ return self.layers(x)+++# --- SAM3.1 Multiplex components ---++class DecoupledMemoryAttnLayer(nn.Module):+ """Decoupled cross-attention layer for SAM3.1: fuses image and memory projections."""++ def __init__(self, d_model=256, num_heads=1, dim_ff=2048, device=None, dtype=None, operations=None):+ super().__init__()+ self.num_heads = num_heads+ # Self-attention projections (flat, not nested)+ self.self_attn_q_proj = operations.Linear(d_model, d_model, device=device, dtype=dtype)+ self.self_attn_k_proj = operations.Linear(d_model, d_model, device=device, dtype=dtype)+ self.self_attn_v_proj = operations.Linear(d_model, d_model, device=device, dtype=dtype)+ self.self_attn_out_proj = operations.Linear(d_model, d_model, device=device, dtype=dtype)+ # Cross-attention projections+ self.cross_attn_q_proj = operations.Linear(d_model, d_model, device=device, dtype=dtype)+ self.cross_attn_k_proj = operations.Linear(d_model, d_model, device=device, dtype=dtype)+ self.cross_attn_v_proj = operations.Linear(d_model, d_model, device=device, dtype=dtype)+ self.cross_attn_out_proj = operations.Linear(d_model, d_model, device=device, dtype=dtype)+ # Image cross-attention (q/k only, fused with cross_attn)+ self.image_cross_attn_q_proj = operations.Linear(d_model, d_model, device=device, dtype=dtype)+ self.image_cross_attn_k_proj = operations.Linear(d_model, d_model, device=device, dtype=dtype)+ # FFN+ self.linear1 = operations.Linear(d_model, dim_ff, device=device, dtype=dtype)+ self.linear2 = operations.Linear(dim_ff, d_model, device=device, dtype=dtype)+ self.norm1 = operations.LayerNorm(d_model, device=device, dtype=dtype)+ self.norm2 = operations.LayerNorm(d_model, device=device, dtype=dtype)+ self.norm3 = operations.LayerNorm(d_model, device=device, dtype=dtype)++ def forward(self, image, x, memory_image, memory, memory_image_pos=None,+ rope=None, num_k_exclude_rope=0):+ # Self-attention with RoPE+ normed = self.norm1(x)+ q = self.self_attn_q_proj(normed)+ k = self.self_attn_k_proj(normed)+ v = self.self_attn_v_proj(normed)+ if rope is not None:+ q, k = apply_rope_memory(q, k, rope, self.num_heads, 0)+ x = x + self.self_attn_out_proj(optimized_attention(q, k, v, self.num_heads))++ # Decoupled cross-attention: fuse image and memory projections+ normed = self.norm2(x)+ q = self.image_cross_attn_q_proj(image) + self.cross_attn_q_proj(normed)+ k = self.image_cross_attn_k_proj(memory_image) + self.cross_attn_k_proj(memory)+ if memory_image_pos is not None:+ k = k + memory_image_pos+ v = self.cross_attn_v_proj(memory)+ if rope is not None:+ q, k = apply_rope_memory(q, k, rope, self.num_heads, num_k_exclude_rope)+ x = x + self.cross_attn_out_proj(optimized_attention(q, k, v, self.num_heads))++ # FFN+ x = x + self.linear2(F.gelu(self.linear1(self.norm3(x))))+ return image, x+++class DecoupledMemoryEncoder(nn.Module):+ """Memory attention encoder for SAM3.1 with decoupled cross-attention."""++ def __init__(self, d_model=256, num_heads=1, dim_ff=2048, num_layers=4, image_size=1008, patch_size=14,+ device=None, dtype=None, operations=None):+ super().__init__()+ self.layers = nn.ModuleList([+ DecoupledMemoryAttnLayer(d_model, num_heads, dim_ff, device=device, dtype=dtype, operations=operations)+ for _ in range(num_layers)+ ])+ self.norm = operations.LayerNorm(d_model, device=device, dtype=dtype)+ hw = image_size // patch_size+ self.register_buffer("_rope", rope_2d(hw, hw, d_model // num_heads), persistent=False)++ def forward(self, x, memory, memory_pos=None, src_pos=None, num_k_exclude_rope=0,+ memory_image=None, memory_image_pos=None):+ image = x # constant residual for decoupled cross-attention+ output = x+ if src_pos is not None:+ output = output + 0.1 * src_pos++ B, _, C = x.shape+ rope = self._rope.to(device=x.device)++ # memory_image: raw backbone features from past frames for decoupled cross-attention+ if memory_image is None:+ # Fallback: use spatial portion of memory (without obj pointers)+ num_spatial = memory.shape[1] - num_k_exclude_rope+ memory_image = memory[:, :num_spatial]+ memory_image_pos = memory_pos[:, :num_spatial] if memory_pos is not None else None+ # Pad memory_image to match memory length (zeros for obj pointer tokens)+ if memory_image.shape[1] < memory.shape[1]:+ pad_len = memory.shape[1] - memory_image.shape[1]+ pad = torch.zeros(B, pad_len, C, device=memory.device, dtype=memory.dtype)+ memory_image = torch.cat([memory_image, pad], dim=1)+ if memory_image_pos is not None:+ ptr_pos = memory_pos[:, -pad_len:] if memory_pos is not None else torch.zeros_like(pad)+ memory_image_pos = torch.cat([memory_image_pos, ptr_pos], dim=1)++ for layer in self.layers:+ image, output = layer(image, output, memory_image, memory,+ memory_image_pos=memory_image_pos, rope=rope,+ num_k_exclude_rope=num_k_exclude_rope)++ return self.norm(output)+++class DecoupledMemoryTransformer(nn.Module):+ def __init__(self, d_model=256, num_heads=1, dim_ff=2048, num_layers=4, device=None, dtype=None, operations=None):+ super().__init__()+ self.encoder = DecoupledMemoryEncoder(d_model, num_heads, dim_ff, num_layers,+ device=device, dtype=dtype, operations=operations)+++class MemoryBackbone(nn.Module):+ """Memory encoder: downsamples mask, fuses with pixel features, optionally compresses."""++ def __init__(self, d_model=256, out_dim=None, in_chans=1, channels=None, device=None, dtype=None, operations=None):+ super().__init__()+ self.mask_downsampler = MaskDownSampler(d_model, in_chans=in_chans, channels=channels, device=device, dtype=dtype, operations=operations)+ self.pix_feat_proj = operations.Conv2d(d_model, d_model, kernel_size=1, device=device, dtype=dtype)+ self.fuser = Fuser(d_model, num_layers=2, device=device, dtype=dtype, operations=operations)+ self.has_out_proj = out_dim is not None and out_dim != d_model+ if self.has_out_proj:+ self.out_proj = operations.Conv2d(d_model, out_dim, kernel_size=1, device=device, dtype=dtype)+ feat_dim = out_dim+ else:+ feat_dim = d_model+ self.position_encoding = PositionEmbeddingSine(num_pos_feats=feat_dim, normalize=True)++ def forward(self, image_features, mask_for_mem, skip_mask_sigmoid=False):+ if not skip_mask_sigmoid:+ mask_for_mem = mask_for_mem.sigmoid()+ mask_features = self.mask_downsampler(cast_to_input(mask_for_mem, image_features))+ if mask_features.shape[-2:] != image_features.shape[-2:]:+ mask_features = F.interpolate(mask_features, size=image_features.shape[-2:], mode="bilinear", align_corners=False)+ features = self.pix_feat_proj(image_features) + mask_features+ features = self.fuser(features)+ if self.has_out_proj:+ features = self.out_proj(features)+ pos = cast_to_input(self.position_encoding(features), features)+ return {"vision_features": features, "vision_pos_enc": [pos]}+++class MultiplexMaskDecoder(nn.Module):+ """SAM mask decoder for SAM3.1 multiplex: predicts masks for num_multiplex objects simultaneously.++ Uses multimask_outputs_only=True: num_mask_output_per_object = num_multimask_outputs (no +1).+ Hypernetwork MLPs are shared across multiplex objects.+ Token order: [obj_score_token(M), iou_token(M), mask_tokens(M*T)].+ """++ def __init__(self, d_model=256, num_multiplex=16, num_multimask_outputs=3, device=None, dtype=None, operations=None):+ super().__init__()+ self.num_multiplex = num_multiplex+ self.num_mask_output_per_object = num_multimask_outputs # 3 (multimask_outputs_only)+ total_mask_tokens = num_multiplex * self.num_mask_output_per_object # 48++ self.transformer = SAMTwoWayTransformer(depth=2, embedding_dim=d_model, num_heads=8, mlp_dim=2048, device=device, dtype=dtype, operations=operations)++ self.obj_score_token = operations.Embedding(num_multiplex, d_model, device=device, dtype=dtype)+ self.iou_token = operations.Embedding(num_multiplex, d_model, device=device, dtype=dtype)+ self.mask_tokens = operations.Embedding(total_mask_tokens, d_model, device=device, dtype=dtype)++ LN2d = LayerNorm2d_op(operations)+ self.output_upscaling = nn.Sequential(+ operations.ConvTranspose2d(d_model, d_model // 4, kernel_size=2, stride=2, device=device, dtype=dtype),+ LN2d(d_model // 4, device=device, dtype=dtype), nn.GELU(),+ operations.ConvTranspose2d(d_model // 4, d_model // 8, kernel_size=2, stride=2, device=device, dtype=dtype), nn.GELU(),+ )+ self.conv_s0 = operations.Conv2d(d_model, d_model // 8, kernel_size=1, device=device, dtype=dtype)+ self.conv_s1 = operations.Conv2d(d_model, d_model // 4, kernel_size=1, device=device, dtype=dtype)++ # Shared across all multiplex objects (one per mask output)+ self.output_hypernetworks_mlps = nn.ModuleList([+ MLP(d_model, d_model, d_model // 8, 3, device=device, dtype=dtype, operations=operations)+ for _ in range(self.num_mask_output_per_object)+ ])+ self.iou_prediction_head = MLP(d_model, d_model, self.num_mask_output_per_object, 3, device=device, dtype=dtype, operations=operations)+ self.pred_obj_score_head = MLP(d_model, d_model, 1, 3, device=device, dtype=dtype, operations=operations)++ def forward(self, image_embeddings, image_pe, sparse_prompt_embeddings, dense_prompt_embeddings,+ high_res_features=None, multimask_output=False, return_all=False, extra_per_object_embeddings=None):+ B = sparse_prompt_embeddings.shape[0]+ M = self.num_multiplex+ T = self.num_mask_output_per_object++ # Token order: [obj_score(M), iou(M), mask(M*T)]+ ref = sparse_prompt_embeddings+ mask_tokens = cast_to_input(self.mask_tokens.weight, ref)+ if extra_per_object_embeddings is not None:+ mask_tokens = mask_tokens.view(1, M, T, -1).expand(B, -1, -1, -1) + extra_per_object_embeddings.unsqueeze(2)+ mask_tokens = mask_tokens.flatten(1, 2) # [B, M*T, C]+ other_tokens = torch.cat([cast_to_input(self.obj_score_token.weight, ref),+ cast_to_input(self.iou_token.weight, ref)], dim=0).unsqueeze(0).expand(B, -1, -1)+ tokens = torch.cat([other_tokens, mask_tokens, sparse_prompt_embeddings], dim=1)+ else:+ tokens = torch.cat([cast_to_input(self.obj_score_token.weight, ref),+ cast_to_input(self.iou_token.weight, ref), mask_tokens], dim=0)+ tokens = torch.cat([tokens.unsqueeze(0).expand(B, -1, -1), sparse_prompt_embeddings], dim=1)++ src = image_embeddings+ if src.shape[0] != B:+ src = src.expand(B, -1, -1, -1)+ src = src + dense_prompt_embeddings+ pos_src = image_pe.expand(B, -1, -1, -1)++ b, c, h, w = src.shape+ hs, src_out = self.transformer(src.flatten(2).permute(0, 2, 1), pos_src.flatten(2).permute(0, 2, 1), tokens)++ # Parse output tokens+ obj_score_token_out = hs[:, :M]+ iou_token_out = hs[:, M:2 * M]+ mask_tokens_out = hs[:, 2 * M:2 * M + M * T]++ src_out = src_out.permute(0, 2, 1).view(b, c, h, w)+ upscaled = _upscale_masks(self.output_upscaling, self.conv_s0, self.conv_s1, src_out, high_res_features)++ # Reshape mask tokens to [B, M, T, C] and apply shared hypernetwork MLPs per mask output index+ mask_tokens_2d = mask_tokens_out.view(B, M, T, -1)+ hyper_in = torch.stack([+ self.output_hypernetworks_mlps[i](mask_tokens_2d[:, :, i, :]) # [B, M, C//8]+ for i in range(T)+ ], dim=2) # [B, M, T, C//8]++ # Generate masks: [B, M*T, H*W] -> [B, M, T, H, W]+ masks = torch.bmm(hyper_in.flatten(1, 2), upscaled.flatten(2)).view(b, M, T, upscaled.shape[2], upscaled.shape[3])++ # IoU and object scores+ iou_pred = self.iou_prediction_head(iou_token_out).view(b, M, T)+ object_score_logits = self.pred_obj_score_head(obj_score_token_out) # [B, M, 1]++ # multimask_outputs_only: always output all T masks (no singlemask token)+ sam_tokens_out = mask_tokens_2d[:, :, 0:1] # [B, M, 1, C]++ if return_all:+ return masks, iou_pred, sam_tokens_out, object_score_logits+ return masks, iou_pred+++class SAM3Tracker(nn.Module):+ def __init__(self, d_model=256, mem_dim=64, num_maskmem=7, device=None, dtype=None, operations=None, **kwargs):+ super().__init__()++ # Memory attention transformer+ self.transformer = MemoryTransformer(d_model, num_heads=1, kv_dim=mem_dim, dim_ff=2048, num_layers=4,+ device=device, dtype=dtype, operations=operations)+ # SAM components+ self.sam_mask_decoder = SAMMaskDecoder(d_model, device=device, dtype=dtype, operations=operations)+ self.sam_prompt_encoder = SAMPromptEncoder(d_model, device=device, dtype=dtype, operations=operations)++ # Memory backbone+ self.maskmem_backbone = MemoryBackbone(d_model, out_dim=mem_dim, device=device, dtype=dtype, operations=operations)++ # Standalone parameters+ self.maskmem_tpos_enc = nn.Parameter(torch.zeros(num_maskmem, 1, 1, mem_dim, device=device, dtype=dtype))+ self.no_mem_embed = nn.Parameter(torch.zeros(1, 1, d_model, device=device, dtype=dtype))+ self.register_buffer("no_mem_pos_enc", torch.zeros(1, 1, d_model, device=device, dtype=dtype)) # checkpoint key, unused in forward+ self.no_obj_embed_spatial = nn.Parameter(torch.zeros(1, mem_dim, device=device, dtype=dtype))+ self.no_obj_ptr = nn.Parameter(torch.zeros(1, d_model, device=device, dtype=dtype))++ # Object pointer projection+ self.obj_ptr_proj = MLP(d_model, d_model, d_model, 3, device=device, dtype=dtype, operations=operations)+ self.obj_ptr_tpos_proj = operations.Linear(d_model, mem_dim, device=device, dtype=dtype)++ # Mask downsample: Conv2d stride 4 to reduce GT mask to SAM logit scale+ self.mask_downsample = operations.Conv2d(1, 1, kernel_size=4, stride=4, device=device, dtype=dtype)++ # Config+ self.d_model = d_model+ self.mem_dim = mem_dim+ self.num_maskmem = num_maskmem+ self.image_size = 1008+ self.backbone_stride = 14+ self.max_obj_ptrs_in_encoder = 16+ self.sigmoid_scale_for_mem_enc = 20.0+ self.sigmoid_bias_for_mem_enc = -10.0++ def _no_obj_blend(self, obj_ptr, is_obj):+ alpha = is_obj.to(obj_ptr.dtype)+ return torch.lerp(cast_to_input(self.no_obj_ptr, obj_ptr), obj_ptr, alpha)++ def _forward_sam_heads(self, backbone_features, point_inputs=None, mask_inputs=None, box_inputs=None,+ high_res_features=None, multimask_output=False):+ return forward_sam_heads(backbone_features, self.sam_prompt_encoder, self.sam_mask_decoder,+ self.obj_ptr_proj, self._no_obj_blend, self.image_size,+ point_inputs, mask_inputs, box_inputs, high_res_features, multimask_output)++ def _use_mask_as_output(self, backbone_features, high_res_features, mask_inputs):+ return use_mask_as_output(backbone_features, high_res_features, mask_inputs,+ self.mask_downsample, self.sam_prompt_encoder, self.sam_mask_decoder,+ self.obj_ptr_proj, self._no_obj_blend, self.image_size, self.backbone_stride)++ def _prepare_memory_conditioned_features(self, frame_idx, is_init_cond_frame, current_vision_feats, current_vision_pos_embeds, feat_sizes, output_dict, num_frames):+ """Fuse current frame features with memory from previous frames."""+ B = current_vision_feats[-1].shape[0]+ C = self.d_model+ H, W = feat_sizes[-1]+ device = current_vision_feats[-1].device++ if self.num_maskmem == 0:+ return current_vision_feats[-1].permute(0, 2, 1).view(B, C, H, W)++ if is_init_cond_frame:+ # First conditioning frame: no memory yet, add no_mem_embed+ pix_feat = current_vision_feats[-1] + cast_to_input(self.no_mem_embed, current_vision_feats[-1])+ return to_spatial(pix_feat, H, W)++ to_cat_memory, to_cat_memory_pos, _, _, cond_outputs = collect_memory_tokens(+ output_dict, frame_idx, self.num_maskmem, self.maskmem_tpos_enc, device)++ max_obj_ptrs = min(num_frames, self.max_obj_ptrs_in_encoder)+ pos_and_ptrs = []+ for t, out in cond_outputs.items():+ if t <= frame_idx:+ pos_and_ptrs.append(((frame_idx - t), out["obj_ptr"].to(device)))+ for t_diff in range(1, max_obj_ptrs):+ t = frame_idx - t_diff+ if t < 0:+ break+ out = output_dict["non_cond_frame_outputs"].get(t, None)+ if out is not None:+ pos_and_ptrs.append((t_diff, out["obj_ptr"].to(device)))++ num_obj_ptr_tokens = 0+ if len(pos_and_ptrs) > 0:+ pos_list, ptrs_list = zip(*pos_and_ptrs)+ obj_ptrs = torch.stack(ptrs_list, dim=1) # [B, N, C=256]++ # Temporal position encoding for pointers+ obj_pos = compute_tpos_enc(+ list(pos_list), device, self.d_model, self.obj_ptr_tpos_proj,+ max_abs_pos=max_obj_ptrs, dtype=current_vision_feats[-1].dtype+ ) # [N, mem_dim=64]+ obj_pos = obj_pos.unsqueeze(0).expand(B, -1, -1) # [B, N, 64]++ # Split each 256-dim pointer into 4 x 64-dim tokens+ if self.mem_dim < C:+ N = obj_ptrs.shape[1]+ obj_ptrs = obj_ptrs.view(B, N, C // self.mem_dim, self.mem_dim) # [B, N, 4, 64]+ obj_ptrs = obj_ptrs.reshape(B, N * (C // self.mem_dim), self.mem_dim) # [B, N*4, 64]+ obj_pos = obj_pos.unsqueeze(2).expand(-1, -1, C // self.mem_dim, -1)+ obj_pos = obj_pos.reshape(B, N * (C // self.mem_dim), self.mem_dim) # [B, N*4, 64]++ to_cat_memory.append(obj_ptrs)+ to_cat_memory_pos.append(obj_pos)+ num_obj_ptr_tokens = obj_ptrs.shape[1]++ if len(to_cat_memory) == 0:+ # No memory available yet, add no_mem_embed+ pix_feat = current_vision_feats[-1] + cast_to_input(self.no_mem_embed, current_vision_feats[-1])+ return to_spatial(pix_feat, H, W)++ # Concatenate all memory and position encodings [B, total_mem, mem_dim=64]+ memory = torch.cat(to_cat_memory, dim=1)+ memory_pos = torch.cat(to_cat_memory_pos, dim=1)++ # Run memory attention encoder+ pix_feat = current_vision_feats[-1] # [B, HW, C]+ src_pos = current_vision_pos_embeds[-1] # [B, HW, C]++ pix_feat_with_mem = self.transformer.encoder(+ x=pix_feat,+ memory=memory,+ src_pos=src_pos,+ memory_pos=memory_pos,+ num_k_exclude_rope=num_obj_ptr_tokens,+ )+ return to_spatial(pix_feat_with_mem, H, W)++ def _encode_new_memory(self, pix_feat, pred_masks_high_res, object_score_logits, is_mask_from_pts=False):+ """Encode predicted mask into memory features."""+ if is_mask_from_pts:+ mask_for_mem = (pred_masks_high_res > 0).to(pix_feat.dtype)+ else:+ mask_for_mem = torch.sigmoid(pred_masks_high_res)++ mask_for_mem.mul_(self.sigmoid_scale_for_mem_enc).add_(self.sigmoid_bias_for_mem_enc)++ maskmem_out = self.maskmem_backbone(pix_feat, mask_for_mem, skip_mask_sigmoid=True)+ maskmem_features = maskmem_out["vision_features"]+ maskmem_pos_enc = maskmem_out["vision_pos_enc"]++ # Add no_obj_embed for occluded objects+ alpha = (object_score_logits > 0).to(maskmem_features.dtype)[..., None, None]+ no_obj = cast_to_input(self.no_obj_embed_spatial, maskmem_features)[..., None, None].expand_as(maskmem_features)+ return maskmem_features + (1 - alpha) * no_obj, maskmem_pos_enc++ def track_step(self, frame_idx, is_init_cond_frame, current_vision_feats, current_vision_pos_embeds, feat_sizes, mask_inputs, output_dict,+ num_frames, point_inputs=None):+ """Track one frame: fuse with memory, predict mask, encode memory."""+ current_out = {}++ # High-res features for SAM head [stride-8, stride-4]+ if len(current_vision_feats) > 1:+ high_res_features = [+ x.view(x.shape[0], feat_sizes[i][0], feat_sizes[i][1], -1).permute(0, 3, 1, 2)+ for i, x in enumerate(current_vision_feats[:-1])+ ]+ else:+ high_res_features = None++ # Top-level feature for memory+ H, W = feat_sizes[-1]++ if mask_inputs is not None:+ # Conditioning frame: use mask directly+ pix_feat = to_spatial(current_vision_feats[-1], H, W)+ sam_outputs = self._use_mask_as_output(pix_feat, high_res_features, mask_inputs)+ else:+ # Track frame: fuse with memory, then SAM decoder+ pix_feat_with_mem = self._prepare_memory_conditioned_features(+ frame_idx=frame_idx,+ is_init_cond_frame=is_init_cond_frame,+ current_vision_feats=current_vision_feats,+ current_vision_pos_embeds=current_vision_pos_embeds,+ feat_sizes=feat_sizes,+ output_dict=output_dict,+ num_frames=num_frames,+ )+ # Use multimask for point prompts on init frames (picks best of 3 candidates)+ num_pts = 0 if point_inputs is None else point_inputs["point_labels"].size(1)+ multimask_output = is_init_cond_frame and 0 < num_pts <= 1+ sam_outputs = self._forward_sam_heads(+ backbone_features=pix_feat_with_mem,+ point_inputs=point_inputs,+ high_res_features=high_res_features,+ multimask_output=multimask_output,+ )++ (low_res_masks, high_res_masks, obj_ptr, object_score_logits) = sam_outputs++ # Clean low-res masks: remove sprinkles and fill holes+ low_res_masks = fill_holes_in_mask_scores(low_res_masks, max_area=200)+ high_res_masks = F.interpolate(low_res_masks, size=(self.image_size, self.image_size), mode="bilinear", align_corners=False)++ current_out["pred_masks"] = low_res_masks+ current_out["pred_masks_high_res"] = high_res_masks+ current_out["obj_ptr"] = obj_ptr+ current_out["object_score_logits"] = object_score_logits++ # Encode memory+ if self.num_maskmem > 0:+ pix_feat = to_spatial(current_vision_feats[-1], H, W)+ maskmem_features, maskmem_pos_enc = self._encode_new_memory(+ pix_feat=pix_feat,+ pred_masks_high_res=high_res_masks,+ object_score_logits=object_score_logits,+ is_mask_from_pts=(point_inputs is not None),+ )+ current_out["maskmem_features"] = maskmem_features+ current_out["maskmem_pos_enc"] = maskmem_pos_enc+ else:+ current_out["maskmem_features"] = None+ current_out["maskmem_pos_enc"] = None++ return current_out++ def _compute_backbone_frame(self, backbone_fn, frame, frame_idx=None):+ vision_feats, vision_pos, feat_sizes, _, _ = _compute_backbone(backbone_fn, frame, frame_idx)+ # SAM3: drop last FPN level+ return vision_feats[:-1], vision_pos[:-1], feat_sizes[:-1]++ def _track_single_object(self, backbone_fn, images, initial_mask, pbar=None):+ """Track one object, computing backbone per frame to save VRAM."""+ N = images.shape[0]+ device, dt = images.device, images.dtype+ output_dict = {"cond_frame_outputs": {}, "non_cond_frame_outputs": {}}+ all_masks = []++ for frame_idx in tqdm(range(N), desc="tracking"):+ vision_feats, vision_pos, feat_sizes = self._compute_backbone_frame(+ backbone_fn, images[frame_idx:frame_idx + 1], frame_idx=frame_idx)+ mask_input = None+ if frame_idx == 0:+ mask_input = F.interpolate(initial_mask.to(device=device, dtype=dt),+ size=(self.image_size, self.image_size), mode="bilinear", align_corners=False)+ mask_input = (mask_input > 0.5).to(dt)++ current_out = self.track_step(+ frame_idx=frame_idx, is_init_cond_frame=(frame_idx == 0),+ current_vision_feats=vision_feats, current_vision_pos_embeds=vision_pos,+ feat_sizes=feat_sizes, mask_inputs=mask_input, output_dict=output_dict, num_frames=N)++ if frame_idx == 0:+ output_dict["cond_frame_outputs"][frame_idx] = current_out+ else:+ output_dict["non_cond_frame_outputs"][frame_idx] = current_out+ lookback = max(self.num_maskmem, self.max_obj_ptrs_in_encoder)+ for old_idx in list(output_dict["non_cond_frame_outputs"]):+ if old_idx < frame_idx - lookback:+ del output_dict["non_cond_frame_outputs"][old_idx]+ # Move masks to CPU immediately to free VRAM+ all_masks.append(current_out["pred_masks_high_res"].to(comfy.model_management.intermediate_device()))+ if pbar is not None:+ pbar.update(1)++ return torch.cat(all_masks, dim=0) # [N, 1, H, W]++ def track_video(self, backbone_fn, images, initial_masks, pbar=None, **kwargs):+ """Track one or more objects across video frames.++ Args:+ backbone_fn: callable that returns (sam2_features, sam2_positions, trunk_out) for a frame+ images: [N, 3, 1008, 1008] video frames+ initial_masks: [N_obj, 1, H, W] binary masks for first frame (one per object)+ pbar: optional progress bar++ Returns:+ [N, N_obj, image_size, image_size] predicted mask logits per frame per object+ """+ N_obj = initial_masks.shape[0]+ per_object = []+ for obj_idx in range(N_obj):+ obj_masks = self._track_single_object(+ backbone_fn, images, initial_masks[obj_idx:obj_idx + 1], pbar=pbar)+ per_object.append(obj_masks)++ return torch.cat(per_object, dim=1) # [N, N_obj, H, W]+++class SAM31Tracker(nn.Module):+ """SAM3.1 multiplex tracker: decoupled memory attention, dual decoder, 16-object multiplex."""++ def __init__(self, d_model=256, mem_dim=256, num_maskmem=7, num_multiplex=16, device=None, dtype=None, operations=None, **kwargs):+ super().__init__()+ self.d_model = d_model+ self.mem_dim = mem_dim+ self.num_maskmem = num_maskmem+ self.num_multiplex = num_multiplex+ self.image_size = 1008+ self.backbone_stride = 14+ self.max_obj_ptrs_in_encoder = 16+ self.sigmoid_scale_for_mem_enc = 2.0+ self.sigmoid_bias_for_mem_enc = -1.0++ # Memory attention (decoupled cross-attention, 8 heads matching reference)+ self.transformer = DecoupledMemoryTransformer(d_model, num_heads=8, dim_ff=2048, num_layers=4,+ device=device, dtype=dtype, operations=operations)++ # Propagation decoder (multiplex: 16 objects, multimask_outputs_only)+ self.sam_mask_decoder = MultiplexMaskDecoder(d_model, num_multiplex, num_multimask_outputs=3,+ device=device, dtype=dtype, operations=operations)+ # Interactive decoder (single object, same as SAM3)+ self.interactive_sam_mask_decoder = SAMMaskDecoder(d_model, num_multimask_outputs=3,+ device=device, dtype=dtype, operations=operations)+ self.interactive_sam_prompt_encoder = SAMPromptEncoder(d_model, device=device, dtype=dtype, operations=operations)++ # Memory backbone (mem_dim=256, no out_proj compression)+ self.maskmem_backbone = MemoryBackbone(d_model, in_chans=num_multiplex * 2, channels=[16, 64, 256, 1024],+ device=device, dtype=dtype, operations=operations)++ # Standalone parameters+ self.maskmem_tpos_enc = nn.Parameter(torch.zeros(num_maskmem, 1, 1, mem_dim, device=device, dtype=dtype))+ self.no_obj_embed_spatial = nn.Parameter(torch.zeros(num_multiplex, mem_dim, device=device, dtype=dtype))+ self.interactivity_no_mem_embed = nn.Parameter(torch.zeros(1, 1, d_model, device=device, dtype=dtype))++ # Object pointer projection+ self.obj_ptr_proj = MLP(d_model, d_model, d_model, 3, device=device, dtype=dtype, operations=operations)+ self.obj_ptr_tpos_proj = operations.Linear(d_model, mem_dim, device=device, dtype=dtype)+ self.no_obj_ptr_linear = operations.Linear(d_model, d_model, device=device, dtype=dtype)+ self.interactive_obj_ptr_proj = MLP(d_model, d_model, d_model, 3, device=device, dtype=dtype, operations=operations)++ # Interactive mask downsample+ self.interactive_mask_downsample = operations.Conv2d(1, 1, kernel_size=4, stride=4, device=device, dtype=dtype)++ # Multiplex validity embeddings+ self.output_valid_embed = nn.Parameter(torch.zeros(num_multiplex, d_model, device=device, dtype=dtype))+ self.output_invalid_embed = nn.Parameter(torch.zeros(num_multiplex, d_model, device=device, dtype=dtype))++ # Position encoding for image (used by multiplex decoder)+ self.image_pe_layer = PositionEmbeddingRandom(d_model // 2)++ def _no_obj_blend(self, obj_ptr, is_obj):+ alpha = is_obj.to(obj_ptr.dtype)+ return torch.lerp(self.no_obj_ptr_linear(obj_ptr), obj_ptr, alpha)++ def _forward_sam_heads(self, backbone_features, point_inputs=None, mask_inputs=None, box_inputs=None,+ high_res_features=None, multimask_output=False):+ return forward_sam_heads(backbone_features, self.interactive_sam_prompt_encoder, self.interactive_sam_mask_decoder,+ self.interactive_obj_ptr_proj, self._no_obj_blend, self.image_size,+ point_inputs, mask_inputs, box_inputs, high_res_features, multimask_output)++ def _use_mask_as_output(self, backbone_features, high_res_features, mask_inputs):+ return use_mask_as_output(backbone_features, high_res_features, mask_inputs,+ self.interactive_mask_downsample, self.interactive_sam_prompt_encoder,+ self.interactive_sam_mask_decoder, self.interactive_obj_ptr_proj,+ self._no_obj_blend, self.image_size, self.backbone_stride)++ def _prepare_memory_conditioned_features(self, frame_idx, is_init_cond_frame, current_vision_feats,+ current_vision_pos_embeds, feat_sizes, output_dict, num_frames,+ multiplex_state=None):+ B = current_vision_feats[-1].shape[0]+ C = self.d_model+ H, W = feat_sizes[-1]+ device = current_vision_feats[-1].device+ num_buc = multiplex_state.num_buckets if multiplex_state is not None else None++ if self.num_maskmem == 0:+ return current_vision_feats[-1].permute(0, 2, 1).view(B, C, H, W)++ if is_init_cond_frame:+ pix_feat = current_vision_feats[-1] + cast_to_input(self.interactivity_no_mem_embed, current_vision_feats[-1])+ return to_spatial(pix_feat, H, W)++ to_cat_memory, to_cat_memory_pos, to_cat_image_feat, to_cat_image_pos, cond_outputs = collect_memory_tokens(+ output_dict, frame_idx, self.num_maskmem, self.maskmem_tpos_enc, device,+ collect_image_feats=True, tpos_v2=True, num_buckets=num_buc)++ max_obj_ptrs = min(num_frames, self.max_obj_ptrs_in_encoder)+ pos_and_ptrs = []+ for t, out in cond_outputs.items():+ if t <= frame_idx and "obj_ptr" in out:+ ptr = out["obj_ptr"].to(device)+ if num_buc is not None:+ ptr = _pad_to_buckets(ptr, num_buc)+ pos_and_ptrs.append(((frame_idx - t), ptr))+ for t_diff in range(1, max_obj_ptrs):+ t = frame_idx - t_diff+ if t < 0:+ break+ out = output_dict["non_cond_frame_outputs"].get(t, None)+ if out is not None and "obj_ptr" in out:+ ptr = out["obj_ptr"].to(device)+ if num_buc is not None:+ ptr = _pad_to_buckets(ptr, num_buc)+ pos_and_ptrs.append((t_diff, ptr))++ num_obj_ptr_tokens = 0+ if len(pos_and_ptrs) > 0:+ pos_list, ptrs_list = zip(*pos_and_ptrs)+ obj_ptrs = torch.stack(ptrs_list, dim=1) # [num_buckets, N, M, C]+ B_ptr = obj_ptrs.shape[0]+ N_ptrs = obj_ptrs.shape[1]+ M = obj_ptrs.shape[2]+ obj_ptrs = obj_ptrs.reshape(B_ptr, N_ptrs * M, -1)+ obj_pos = compute_tpos_enc(list(pos_list), device, self.d_model, self.obj_ptr_tpos_proj,+ max_abs_pos=max_obj_ptrs, dtype=current_vision_feats[-1].dtype)+ obj_pos = obj_pos.unsqueeze(0).expand(B_ptr, -1, -1)+ obj_pos = obj_pos.unsqueeze(2).expand(-1, -1, M, -1).reshape(B_ptr, N_ptrs * M, -1)+ to_cat_memory.append(obj_ptrs)+ to_cat_memory_pos.append(obj_pos)+ num_obj_ptr_tokens = obj_ptrs.shape[1]++ if len(to_cat_memory) == 0:+ pix_feat = current_vision_feats[-1] + cast_to_input(self.interactivity_no_mem_embed, current_vision_feats[-1])+ return to_spatial(pix_feat, H, W)++ memory = torch.cat(to_cat_memory, dim=1)+ memory_pos = torch.cat(to_cat_memory_pos, dim=1)++ # Expand vision features to num_buckets if memory has more buckets than B+ mem_B = memory.shape[0]+ x = current_vision_feats[-1]+ x_pos = current_vision_pos_embeds[-1]+ if x.shape[0] < mem_B:+ x = x.expand(mem_B, -1, -1)+ x_pos = x_pos.expand(mem_B, -1, -1)++ if len(to_cat_image_feat) > 0:+ # Decoupled cross-attention: separate image features from memory+ memory_image = cast_to_input(torch.cat(to_cat_image_feat, dim=1), x)+ memory_image_pos = cast_to_input(torch.cat(to_cat_image_pos, dim=1), x)+ if memory_image.shape[0] < mem_B:+ memory_image = memory_image.expand(mem_B, -1, -1)+ memory_image_pos = memory_image_pos.expand(mem_B, -1, -1)+ pix_feat_with_mem = self.transformer.encoder(+ x=x,+ memory=cast_to_input(memory, x),+ memory_pos=cast_to_input(memory_pos, x),+ src_pos=cast_to_input(x_pos, x),+ num_k_exclude_rope=num_obj_ptr_tokens,+ memory_image=memory_image,+ memory_image_pos=memory_image_pos,+ )+ else:+ pix_feat_with_mem = self.transformer.encoder(+ x=x,+ memory=memory,+ memory_pos=memory_pos,+ src_pos=x_pos,+ num_k_exclude_rope=num_obj_ptr_tokens,+ )+ return to_spatial(pix_feat_with_mem, H, W)++ def _encode_new_memory(self, pix_feat, pred_masks_high_res, object_score_logits, is_mask_from_pts=False,+ multiplex_state=None, is_conditioning=False, cond_obj_mask=None):+ if is_mask_from_pts:+ mask_for_mem = (pred_masks_high_res > 0).to(pix_feat.dtype)+ else:+ mask_for_mem = torch.sigmoid(pred_masks_high_res)+ mask_for_mem.mul_(self.sigmoid_scale_for_mem_enc).add_(self.sigmoid_bias_for_mem_enc)++ # Mux masks: [N_obj, 1, H, W] -> [num_buckets, M, H, W]+ mux_masks = multiplex_state.mux(mask_for_mem[:, 0])++ # Conditioning channel: 1.0 = clean mask (trust it), 0.0 = propagation (noisy)+ N_obj = mask_for_mem.shape[0]+ cond_values = torch.full((N_obj,), 0.0, device=mask_for_mem.device, dtype=mask_for_mem.dtype)+ if is_conditioning:+ cond_values[:] = 1.0+ elif cond_obj_mask is not None:+ cond_values[cond_obj_mask] = 1.0+ cond_spatial = cond_values.view(-1, 1, 1, 1).expand_as(mask_for_mem[:, 0:1, :, :]).squeeze(1)+ mux_cond = multiplex_state.mux(cond_spatial) # [num_buckets, M, H, W]+ mux_input = torch.cat([mux_masks, mux_cond], dim=1) # [num_buckets, 2*M, H, W]++ maskmem_out = self.maskmem_backbone(pix_feat, mux_input, skip_mask_sigmoid=True)+ maskmem_features = maskmem_out["vision_features"]+ maskmem_pos_enc = maskmem_out["vision_pos_enc"]++ # Add no_obj_embed_spatial for occluded objects+ is_obj = (object_score_logits > 0).float() # [N_obj, 1]+ mux_is_obj = multiplex_state.mux(is_obj) # [num_buckets, M, 1]+ no_obj_embed = cast_to_input(self.no_obj_embed_spatial, maskmem_features) # [M, C]+ no_obj_spatial = no_obj_embed.unsqueeze(0)[..., None, None] # [1, M, C, 1, 1]+ # Expand and sum across multiplex slots weighted by (1 - is_obj)+ alpha = mux_is_obj[..., None, None] # [num_buckets, M, 1, 1, 1]+ per_slot_no_obj = ((1 - alpha) * no_obj_spatial).sum(dim=1) # [num_buckets, C, 1, 1]+ maskmem_features = maskmem_features + per_slot_no_obj.expand_as(maskmem_features)++ return maskmem_features, maskmem_pos_enc++ def _forward_propagation(self, backbone_features, high_res_features=None, multiplex_state=None):+ """Propagation path using the multiplex SAM decoder (no prompts)."""+ B = backbone_features.shape[0]+ device = backbone_features.device++ # Suppression embeddings from valid object mask+ valid_mask = cast_to_input(multiplex_state.get_valid_object_mask().unsqueeze(-1).float(), backbone_features)+ output_valid = cast_to_input(self.output_valid_embed, backbone_features).unsqueeze(0)+ output_invalid = cast_to_input(self.output_invalid_embed, backbone_features).unsqueeze(0)+ extra_embed = valid_mask * output_valid + (1 - valid_mask) * output_invalid++ image_pe = self.image_pe_layer((backbone_features.shape[-2], backbone_features.shape[-1]), device=backbone_features.device)+ image_pe = cast_to_input(image_pe, backbone_features)++ masks, iou_pred, sam_tokens_out, object_score_logits = self.sam_mask_decoder(+ image_embeddings=backbone_features, image_pe=image_pe,+ sparse_prompt_embeddings=torch.empty(B, 0, self.d_model, device=device, dtype=backbone_features.dtype),+ dense_prompt_embeddings=torch.zeros(B, self.d_model, *backbone_features.shape[-2:], device=device, dtype=backbone_features.dtype),+ high_res_features=high_res_features, multimask_output=True, return_all=True,+ extra_per_object_embeddings=extra_embed.expand(B, -1, -1),+ )+ # masks: [B=num_buckets, M, T, H, W]+ # Demux to per-object: [N_obj, T, H, W]+ masks_obj = multiplex_state.demux(masks)+ iou_obj = multiplex_state.demux(iou_pred)+ score_obj = multiplex_state.demux(object_score_logits)+ tokens_obj = multiplex_state.demux(sam_tokens_out)++ # Select best mask by IoU for each object+ best_idx = torch.argmax(iou_obj, dim=-1) # [N_obj]+ N_obj = masks_obj.shape[0]+ obj_range = torch.arange(N_obj, device=device)+ low_res_masks = masks_obj[obj_range, best_idx].unsqueeze(1) # [N_obj, 1, H, W]+ # Suppress masks for objects with low confidence+ is_obj = score_obj > 0+ low_res_masks = torch.where(is_obj[:, :, None, None], low_res_masks,+ torch.tensor(NO_OBJ_SCORE, device=device, dtype=low_res_masks.dtype))+ high_res_masks = F.interpolate(low_res_masks.float(), size=(self.image_size, self.image_size), mode="bilinear", align_corners=False)++ # Object pointer: compute per-object, mux for storage as [num_buckets, M, C]+ sam_token = tokens_obj[:, 0] # [N_obj, C]+ obj_ptr = self.obj_ptr_proj(sam_token)+ is_obj = (score_obj > 0).float()+ no_obj = self.no_obj_ptr_linear(obj_ptr)+ obj_ptr = is_obj * obj_ptr + (1 - is_obj) * no_obj+ obj_ptr_muxed = multiplex_state.mux(obj_ptr) # [num_buckets, M, C]++ return low_res_masks, high_res_masks, obj_ptr_muxed, score_obj++ def track_step(self, frame_idx, is_init_cond_frame, current_vision_feats, current_vision_pos_embeds,+ feat_sizes, mask_inputs, output_dict, num_frames, point_inputs=None,+ interactive_high_res=None, interactive_backbone=None, propagation_high_res=None,+ multiplex_state=None, run_mem_encoder=True):+ current_out = {}+ H, W = feat_sizes[-1]++ if mask_inputs is not None:+ # Conditioning frame: use interactive features if available, else propagation+ if interactive_backbone is not None:+ pix_feat = interactive_backbone+ # Add no_mem_embed for interactive path+ pix_flat = pix_feat.flatten(2)+ bf = pix_flat.permute(0, 2, 1) + cast_to_input(self.interactivity_no_mem_embed, pix_flat)+ pix_feat = to_spatial(bf, H, W)+ hi_res = interactive_high_res+ else:+ # Fallback: interactive backbone not available (e.g. called outside track_video).+ # Propagation features work but may produce lower-quality conditioning.+ pix_feat = to_spatial(current_vision_feats[-1], H, W)+ hi_res = propagation_high_res+ sam_outputs = self._use_mask_as_output(pix_feat, hi_res, mask_inputs)+ elif point_inputs is not None:+ # Interactive path: use interactive SAM decoder+ pix_feat_with_mem = self._prepare_memory_conditioned_features(+ frame_idx=frame_idx, is_init_cond_frame=is_init_cond_frame,+ current_vision_feats=current_vision_feats, current_vision_pos_embeds=current_vision_pos_embeds,+ feat_sizes=feat_sizes, output_dict=output_dict, num_frames=num_frames,+ multiplex_state=multiplex_state,+ )+ hi_res = interactive_high_res if interactive_high_res is not None else propagation_high_res+ num_pts = point_inputs["point_labels"].size(1)+ multimask_output = is_init_cond_frame and 0 < num_pts <= 1+ sam_outputs = self._forward_sam_heads(+ backbone_features=pix_feat_with_mem, point_inputs=point_inputs,+ high_res_features=hi_res, multimask_output=multimask_output,+ )+ else:+ # Propagation path: use multiplex SAM decoder with propagation features+ pix_feat_with_mem = self._prepare_memory_conditioned_features(+ frame_idx=frame_idx, is_init_cond_frame=is_init_cond_frame,+ current_vision_feats=current_vision_feats, current_vision_pos_embeds=current_vision_pos_embeds,+ feat_sizes=feat_sizes, output_dict=output_dict, num_frames=num_frames,+ multiplex_state=multiplex_state,+ )+ sam_outputs = self._forward_propagation(pix_feat_with_mem, propagation_high_res,+ multiplex_state=multiplex_state)++ (low_res_masks, high_res_masks, obj_ptr, object_score_logits) = sam_outputs++ # Mux obj_ptr if it came from interactive path (shape [B, C]) vs propagation ([num_buckets, M, C])+ if multiplex_state is not None and obj_ptr.dim() == 2:+ obj_ptr = multiplex_state.mux(obj_ptr) # [N_obj, C] -> [num_buckets, M, C]++ # Encode memory (can be deferred with run_mem_encoder=False)+ if run_mem_encoder and self.num_maskmem > 0:+ pix_feat = to_spatial(current_vision_feats[-1], H, W)+ maskmem_features, maskmem_pos_enc = self._encode_new_memory(+ pix_feat=pix_feat, pred_masks_high_res=high_res_masks,+ object_score_logits=object_score_logits,+ is_mask_from_pts=(point_inputs is not None),+ multiplex_state=multiplex_state,+ is_conditioning=(mask_inputs is not None),+ )+ current_out["maskmem_features"] = maskmem_features+ current_out["maskmem_pos_enc"] = maskmem_pos_enc+ else:+ current_out["maskmem_features"] = None+ current_out["maskmem_pos_enc"] = None++ # Store propagation image features for decoupled memory attention+ current_out["image_features"] = current_vision_feats[-1] # [B, HW, C]+ current_out["image_pos_enc"] = current_vision_pos_embeds[-1] # [B, HW, C]++ current_out["pred_masks"] = low_res_masks+ current_out["pred_masks_high_res"] = high_res_masks+ current_out["obj_ptr"] = obj_ptr+ current_out["object_score_logits"] = object_score_logits++ return current_out++ def _compute_backbone_frame(self, backbone_fn, frame, frame_idx=None):+ vision_feats, vision_pos, feat_sizes, features, trunk_out = _compute_backbone(backbone_fn, frame, frame_idx)+ return vision_feats, vision_pos, feat_sizes, list(features[:-1]), trunk_out++ @staticmethod+ def _suppress_recently_occluded(low_res_masks, last_occluded, frame_idx, threshold=0.3):+ """Suppress overlapping masks for objects that were most recently occluded.+ Prevents corrupted masks from occluded objects from contaminating other objects."""+ N_obj = low_res_masks.shape[0]+ if N_obj <= 1:+ return low_res_masks+ binary = low_res_masks[:, 0] > 0 # [N_obj, H, W]+ iou = _compute_mask_overlap(low_res_masks[:, 0], low_res_masks[:, 0])+ overlapping = torch.triu(iou >= threshold, diagonal=1) # [N, N] upper triangle+ last_occ_i = last_occluded.unsqueeze(1) # [N, 1]+ last_occ_j = last_occluded.unsqueeze(0) # [1, N]+ # Suppress the more recently occluded object in each overlapping pair+ suppress_i = overlapping & (last_occ_i > last_occ_j) & (last_occ_j > -1)+ suppress_j = overlapping & (last_occ_j > last_occ_i) & (last_occ_i > -1)+ to_suppress = suppress_i.any(dim=1) | suppress_j.any(dim=0)+ # Update last_occluded for occluded/suppressed objects+ is_empty = ~binary.any(dim=(-1, -2))+ newly_occluded = is_empty | to_suppress+ last_occluded[newly_occluded] = frame_idx+ # Suppress masks+ low_res_masks[to_suppress] = -10.0+ return low_res_masks++ def _deferred_memory_encode(self, current_out, N_obj, vision_feats, feat_sizes, mux_state, device,+ cond_obj_mask=None):+ """Deferred memory encoding for propagation frames. cond_obj_mask: per-object bool for conditioning."""+ low_res_masks = current_out["pred_masks"] # [N_obj, 1, H_low, W_low]++ if N_obj > 1:+ lr = low_res_masks.squeeze(1) # [N_obj, H, W]+ max_obj = torch.argmax(lr, dim=0, keepdim=True)+ batch_inds = torch.arange(N_obj, device=device)[:, None, None]+ pixel_nol = torch.where(max_obj == batch_inds, lr, torch.clamp(lr, max=-10.0))+ area_before = (lr > 0).sum(dim=(-1, -2)).float().clamp(min=1)+ area_after = (pixel_nol > 0).sum(dim=(-1, -2)).float()+ shrink_ok = (area_after / area_before) >= 0.3+ low_res_masks = torch.where(+ shrink_ok[:, None, None, None].expand_as(low_res_masks),+ low_res_masks, torch.clamp(low_res_masks, max=-10.0))++ interpol_size = self.maskmem_backbone.mask_downsampler.interpol_size+ mem_masks = F.interpolate(low_res_masks, size=interpol_size,+ mode="bilinear", align_corners=False)++ obj_scores = torch.where(+ (mem_masks > 0).any(dim=(-1, -2)), 10.0, -10.0)++ pix_feat = to_spatial(vision_feats[-1], feat_sizes[-1][0], feat_sizes[-1][1])+ maskmem_features, maskmem_pos_enc = self._encode_new_memory(+ pix_feat=pix_feat, pred_masks_high_res=mem_masks,+ object_score_logits=obj_scores,+ multiplex_state=mux_state, cond_obj_mask=cond_obj_mask)+ current_out["maskmem_features"] = maskmem_features+ current_out["maskmem_pos_enc"] = maskmem_pos_enc++ def _add_detected_objects(self, new_masks, mux_state, vision_feats, feat_sizes, current_out):+ """Grow MultiplexState with new detections, merge masks, re-encode memory. Modifies current_out."""+ n_old = mux_state.total_valid_entries+ mux_state.add_objects(new_masks.shape[0])+ N_obj = mux_state.total_valid_entries+ # Stored memory with old bucket counts is padded at read time by _pad_to_buckets+ for k in ("pred_masks", "pred_masks_high_res"):+ det = F.interpolate(new_masks.unsqueeze(1), size=current_out[k].shape[-2:],+ mode="bilinear", align_corners=False)+ current_out[k] = torch.cat([current_out[k], det], dim=0)+ if self.num_maskmem > 0:+ # Mark new objects as conditioning (clean detection masks) so model trusts them+ cond_mask = torch.zeros(N_obj, dtype=torch.bool, device=new_masks.device)+ cond_mask[n_old:] = True+ self._deferred_memory_encode(current_out, N_obj, vision_feats, feat_sizes,+ mux_state, new_masks.device, cond_obj_mask=cond_mask)++ def _condition_with_masks(self, masks, frame_idx, vision_feats, vision_pos, feat_sizes,+ high_res_prop, output_dict, N, mux_state, backbone_obj, frame,+ trunk_out, threshold=0.5):+ """Condition tracker with masks on a frame."""+ mask_input = F.interpolate(masks if masks.dim() == 4 else masks.unsqueeze(1),+ size=(self.image_size, self.image_size), mode="bilinear", align_corners=False)+ mask_input = (mask_input > threshold).to(masks.dtype)+ hi_res = lo_feat = None+ if backbone_obj is not None and backbone_obj.multiplex:+ _, _, itf, _ = backbone_obj(frame, tracker_mode="interactive", cached_trunk=trunk_out, tracker_only=True)+ hi_res, lo_feat = itf[:-1], itf[-1]+ current_out = self.track_step(+ frame_idx=frame_idx, is_init_cond_frame=True, current_vision_feats=vision_feats,+ current_vision_pos_embeds=vision_pos, feat_sizes=feat_sizes, mask_inputs=mask_input,+ output_dict=output_dict, num_frames=N, interactive_high_res=hi_res,+ interactive_backbone=lo_feat, propagation_high_res=high_res_prop,+ multiplex_state=mux_state, run_mem_encoder=True)+ output_dict["cond_frame_outputs"][frame_idx] = current_out+ return current_out++ def _match_and_add_detections(self, det_masks, det_scores, current_out, mux_state,+ vision_feats, feat_sizes, device, max_objects=0,+ keep_alive=None):+ """Match detections against tracked masks, add new objects, recondition degraded tracks.+ Updates keep_alive counters: +1 for matched tracks, -1 for unmatched."""+ N_obj = mux_state.total_valid_entries+ if det_masks.shape[0] == 0:+ if keep_alive is not None:+ for i in range(N_obj):+ keep_alive[i] = max(-4, keep_alive.get(i, 0) - 1)+ return []++ # Match at low-res (like reference)+ trk_masks = current_out["pred_masks"][:, 0] # [N_obj, H_low, W_low]+ det_resized = F.interpolate(det_masks.unsqueeze(1), size=trk_masks.shape[-2:],+ mode="bilinear", align_corners=False)[:, 0]+ overlap = _compute_mask_overlap(det_resized, trk_masks)++ # Update keep_alive and find matched tracks+ matched = set()+ if overlap.shape[1] > 0:+ matched = set((overlap >= 0.5).any(dim=0).nonzero(as_tuple=True)[0].tolist())+ if keep_alive is not None:+ for i in range(N_obj):+ if i in matched:+ keep_alive[i] = min(8, keep_alive.get(i, 0) + 1)+ else:+ keep_alive[i] = max(-4, keep_alive.get(i, 0) - 1)++ # Recondition: high-confidence detections (>=0.8) with high overlap refresh tracked masks+ reconditioned = False+ if det_scores is not None and overlap.shape[1] > 0:+ HIGH_CONF = 0.8+ for det_idx in range(overlap.shape[0]):+ if det_scores[det_idx] < HIGH_CONF:+ continue+ best_trk = overlap[det_idx].argmax().item()+ if overlap[det_idx, best_trk] >= 0.5:+ # Replace tracked mask with fresh detection mask+ current_out["pred_masks"][best_trk] = det_resized[det_idx].unsqueeze(0)+ det_hr = F.interpolate(det_masks[det_idx:det_idx+1].unsqueeze(1),+ size=current_out["pred_masks_high_res"].shape[-2:],+ mode="bilinear", align_corners=False)+ current_out["pred_masks_high_res"][best_trk] = det_hr[0]+ reconditioned = True++ # Re-encode memory if any tracks were reconditioned+ if reconditioned and self.num_maskmem > 0:+ self._deferred_memory_encode(current_out, N_obj, vision_feats, feat_sizes, mux_state, device)++ # Add new detections (not matching any track)+ if max_objects > 0 and N_obj >= max_objects:+ return []+ max_overlap = overlap.max(dim=1)[0] if overlap.shape[1] > 0 else torch.zeros(overlap.shape[0], device=device)+ new_dets = max_overlap < 0.5+ if new_dets.any():+ if max_objects > 0:+ slots = max_objects - N_obj+ new_dets = new_dets & (torch.cumsum(new_dets.int(), 0) <= slots)+ self._add_detected_objects(det_masks[new_dets], mux_state,+ vision_feats, feat_sizes, current_out)+ if keep_alive is not None:+ for i in range(N_obj, mux_state.total_valid_entries):+ keep_alive[i] = 1+ return det_scores[new_dets].tolist() if det_scores is not None else [0.0] * new_dets.sum().item()+ return []++ def track_video_with_detection(self, backbone_fn, images, initial_masks, detect_fn=None,+ new_det_thresh=0.5, max_objects=0, detect_interval=1,+ backbone_obj=None, pbar=None):+ """Track with optional per-frame detection. Returns [N, max_N_obj, H, W] mask logits."""+ N, device, dt = images.shape[0], images.device, images.dtype+ output_dict = {"cond_frame_outputs": {}, "non_cond_frame_outputs": {}}+ all_masks = []+ idev = comfy.model_management.intermediate_device()+ mux_state = None+ if initial_masks is not None:+ mux_state = MultiplexState(initial_masks.shape[0], self.num_multiplex, device, dt)+ obj_scores = [] # per-object detection score (1.0 for initial masks)+ keep_alive = {} if detect_fn is not None else None+ last_occluded = torch.empty(0, device=device, dtype=torch.long) # per-object last occluded frame++ # Prefetch next frame's backbone on a separate CUDA stream+ prefetch = False+ backbone_stream = None+ if comfy.model_management.is_device_cuda(device):+ try:+ backbone_stream = torch.cuda.Stream(device=device)+ prefetch = True+ except RuntimeError:+ pass+ cur_bb = self._compute_backbone_frame(backbone_fn, images[0:1], frame_idx=0)++ for frame_idx in tqdm(range(N), desc="tracking"):+ vision_feats, vision_pos, feat_sizes, high_res_prop, trunk_out = cur_bb++ # Start next frame's backbone on separate stream (overlaps with current frame's work)+ if prefetch and frame_idx + 1 < N:+ backbone_stream.wait_stream(torch.cuda.current_stream(device))+ with torch.cuda.stream(backbone_stream):+ next_bb = self._compute_backbone_frame(+ backbone_fn, images[frame_idx + 1:frame_idx + 2], frame_idx=frame_idx + 1)++ # Per-frame detection with NMS (skip if no detect_fn, or interval/max not met)+ det_masks = torch.empty(0, device=device)+ det_scores = None+ run_det = (detect_fn is not None+ and frame_idx % max(detect_interval, 1) == 0+ and not (max_objects > 0 and mux_state is not None+ and mux_state.total_valid_entries >= max_objects))+ if run_det:+ det_out = detect_fn(trunk_out)+ scores = det_out["scores"][0].sigmoid()+ keep = scores > new_det_thresh+ det_masks, det_scores = det_out["masks"][0][keep], scores[keep]+ if det_masks.shape[0] > 1:+ det_masks, det_scores = _nms_masks(det_masks, det_scores)++ if frame_idx == 0 and initial_masks is not None:+ current_out = self._condition_with_masks(+ initial_masks.to(device=device, dtype=dt), frame_idx, vision_feats, vision_pos,+ feat_sizes, high_res_prop, output_dict, N, mux_state, backbone_obj,+ images[frame_idx:frame_idx + 1], trunk_out)+ last_occluded = torch.full((mux_state.total_valid_entries,), -1, device=device, dtype=torch.long)+ obj_scores = [1.0] * mux_state.total_valid_entries+ if keep_alive is not None:+ for i in range(mux_state.total_valid_entries):+ keep_alive[i] = 8+ elif mux_state is None or mux_state.total_valid_entries == 0:+ if det_masks.shape[0] > 0:+ if max_objects > 0:+ det_scores = det_scores[:max_objects]+ det_masks = det_masks[:max_objects]+ mux_state = MultiplexState(det_masks.shape[0], self.num_multiplex, device, dt)+ current_out = self._condition_with_masks(+ det_masks, frame_idx, vision_feats, vision_pos, feat_sizes, high_res_prop,+ output_dict, N, mux_state, backbone_obj,+ images[frame_idx:frame_idx + 1], trunk_out, threshold=0.0)+ last_occluded = torch.full((mux_state.total_valid_entries,), -1, device=device, dtype=torch.long)+ obj_scores = det_scores[:mux_state.total_valid_entries].tolist()+ if keep_alive is not None:+ for i in range(mux_state.total_valid_entries):+ keep_alive[i] = 1+ else:+ all_masks.append(None)+ if pbar is not None:+ pbar.update(1)+ # Skip to backbone advance at end of loop+ if frame_idx + 1 < N:+ if prefetch:+ torch.cuda.current_stream(device).wait_stream(backbone_stream)+ cur_bb = next_bb+ else:+ cur_bb = self._compute_backbone_frame(backbone_fn, images[frame_idx + 1:frame_idx + 2], frame_idx=frame_idx + 1)+ continue+ else:+ N_obj = mux_state.total_valid_entries+ current_out = self.track_step(+ frame_idx=frame_idx, is_init_cond_frame=False, current_vision_feats=vision_feats,+ current_vision_pos_embeds=vision_pos, feat_sizes=feat_sizes, mask_inputs=None,+ output_dict=output_dict, num_frames=N, propagation_high_res=high_res_prop,+ multiplex_state=mux_state, run_mem_encoder=False)+ current_out["pred_masks"] = fill_holes_in_mask_scores(+ current_out["pred_masks"], max_area=16)+ if last_occluded.shape[0] == N_obj and N_obj > 1:+ self._suppress_recently_occluded(+ current_out["pred_masks"], last_occluded, frame_idx)+ if self.num_maskmem > 0:+ self._deferred_memory_encode(current_out, N_obj, vision_feats, feat_sizes, mux_state, device)+ output_dict["non_cond_frame_outputs"][frame_idx] = current_out+ lookback = max(self.num_maskmem, self.max_obj_ptrs_in_encoder)+ for old_idx in list(output_dict["non_cond_frame_outputs"]):+ if old_idx < frame_idx - lookback:+ del output_dict["non_cond_frame_outputs"][old_idx]+ n_before = mux_state.total_valid_entries+ new_obj_scores = self._match_and_add_detections(det_masks, det_scores, current_out, mux_state,+ vision_feats, feat_sizes, device, max_objects,+ keep_alive if run_det else None)+ n_added = mux_state.total_valid_entries - n_before+ if n_added > 0:+ last_occluded = torch.cat([last_occluded,+ torch.full((n_added,), -1, device=device, dtype=torch.long)])+ obj_scores.extend(new_obj_scores)++ masks_out = current_out["pred_masks_high_res"][:, 0]+ if keep_alive is not None:+ for i in range(masks_out.shape[0]):+ if keep_alive.get(i, 0) <= 0:+ masks_out[i] = NO_OBJ_SCORE+ N_obj_now = mux_state.total_valid_entries if mux_state is not None else 0+ if N_obj_now > 0:+ all_masks.append(pack_masks(masks_out).to(idev))+ else:+ all_masks.append(None)+ if pbar is not None:+ pbar.update(1)++ # Next frame's backbone+ if frame_idx + 1 < N:+ if prefetch:+ torch.cuda.current_stream(device).wait_stream(backbone_stream)+ cur_bb = next_bb+ else:+ cur_bb = self._compute_backbone_frame(backbone_fn, images[frame_idx + 1:frame_idx + 2], frame_idx=frame_idx + 1)++ if not all_masks or all(m is None for m in all_masks):+ return {"packed_masks": None, "n_frames": N, "scores": []}++ max_obj = max(m.shape[0] for m in all_masks if m is not None)+ sample = next(m for m in all_masks if m is not None)+ empty_packed = torch.zeros(max_obj, *sample.shape[1:], dtype=torch.uint8, device=sample.device)+ for i, m in enumerate(all_masks):+ if m is None:+ all_masks[i] = empty_packed+ elif m.shape[0] < max_obj:+ pad = torch.zeros(max_obj - m.shape[0], *m.shape[1:], dtype=torch.uint8, device=m.device)+ all_masks[i] = torch.cat([m, pad], dim=0)+ return {"packed_masks": torch.stack(all_masks, dim=0), "n_frames": N, "scores": obj_scores}diff --git a/comfy/model_base.py b/comfy/model_base.pyindex 1c7695761..787ea1145 100644--- a/comfy/model_base.py+++ b/comfy/model_base.py@@ -54,6 +54,7 @@ import comfy.ldm.anima.modelimport comfy.ldm.ace.ace_step15import comfy.ldm.rt_detr.rtdetr_v4import comfy.ldm.ernie.model+import comfy.ldm.sam3.detectorimport comfy.model_managementimport comfy.patcher_extension@@ -1974,3 +1975,7 @@ class ErnieImage(BaseModel):if cross_attn is not None:out['c_crossattn'] = comfy.conds.CONDRegular(cross_attn)return out++class SAM3(BaseModel):+ def __init__(self, model_config, model_type=ModelType.FLOW, device=None):+ super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.sam3.detector.SAM3Model)diff --git a/comfy/model_detection.py b/comfy/model_detection.pyindex ca06cdd1e..724a241bf 100644--- a/comfy/model_detection.py+++ b/comfy/model_detection.py@@ -718,6 +718,14 @@ def detect_unet_config(state_dict, key_prefix, metadata=None):dit_config["image_model"] = "ernie"return dit_config+ if 'detector.backbone.vision_backbone.trunk.blocks.0.attn.qkv.weight' in state_dict_keys: # SAM3 / SAM3.1+ if 'detector.transformer.decoder.query_embed.weight' in state_dict_keys:+ dit_config = {}+ dit_config["image_model"] = "SAM3"+ if 'detector.backbone.vision_backbone.propagation_convs.0.conv_1x1.weight' in state_dict_keys:+ dit_config["image_model"] = "SAM31"+ return dit_config+if '{}input_blocks.0.0.weight'.format(key_prefix) not in state_dict_keys:return None@@ -873,6 +881,10 @@ def model_config_from_unet(state_dict, unet_key_prefix, use_base_if_no_match=Falreturn model_configdef unet_prefix_from_state_dict(state_dict):+ # SAM3: detector.* and tracker.* at top level, no common prefix+ if any(k.startswith("detector.") for k in state_dict) and any(k.startswith("tracker.") for k in state_dict):+ return ""+candidates = ["model.diffusion_model.", #ldm/sgm models"model.model.", #audio models"net.", #cosmosdiff --git a/comfy/supported_models.py b/comfy/supported_models.pyindex 58d4ce731..8886f32d5 100644--- a/comfy/supported_models.py+++ b/comfy/supported_models.py@@ -1781,6 +1781,57 @@ class ErnieImage(supported_models_base.BASE):return supported_models_base.ClipTarget(comfy.text_encoders.ernie.ErnieTokenizer, comfy.text_encoders.ernie.te(**hunyuan_detect))-models = [LotusD, Stable_Zero123, SD15_instructpix2pix, SD15, SD20, SD21UnclipL, SD21UnclipH, SDXL_instructpix2pix, SDXLRefiner, SDXL, SSD1B, KOALA_700M, KOALA_1B, Segmind_Vega, SD_X4Upscaler, Stable_Cascade_C, Stable_Cascade_B, SV3D_u, SV3D_p, SD3, StableAudio, AuraFlow, PixArtAlpha, PixArtSigma, HunyuanDiT, HunyuanDiT1, FluxInpaint, Flux, LongCatImage, FluxSchnell, GenmoMochi, LTXV, LTXAV, HunyuanVideo15_SR_Distilled, HunyuanVideo15, HunyuanImage21Refiner, HunyuanImage21, HunyuanVideoSkyreelsI2V, HunyuanVideoI2V, HunyuanVideo, CosmosT2V, CosmosI2V, CosmosT2IPredict2, CosmosI2VPredict2, ZImagePixelSpace, ZImage, Lumina2, WAN22_T2V, WAN21_T2V, WAN21_I2V, WAN21_FunControl2V, WAN21_Vace, WAN21_Camera, WAN22_Camera, WAN22_S2V, WAN21_HuMo, WAN22_Animate, WAN21_FlowRVS, WAN21_SCAIL, Hunyuan3Dv2mini, Hunyuan3Dv2, Hunyuan3Dv2_1, HiDream, Chroma, ChromaRadiance, ACEStep, ACEStep15, Omnigen2, QwenImage, Flux2, Kandinsky5Image, Kandinsky5, Anima, RT_DETR_v4, ErnieImage]+class SAM3(supported_models_base.BASE):+ unet_config = {"image_model": "SAM3"}+ supported_inference_dtypes = [torch.float16, torch.bfloat16, torch.float32]+ text_encoder_key_prefix = ["detector.backbone.language_backbone."]+ unet_extra_prefix = ""++ def process_clip_state_dict(self, state_dict):+ clip_keys = getattr(self, "_clip_stash", {})+ clip_keys = utils.state_dict_prefix_replace(clip_keys, {"detector.backbone.language_backbone.": "", "backbone.language_backbone.": ""}, filter_keys=True)+ clip_keys = utils.clip_text_transformers_convert(clip_keys, "encoder.", "sam3_clip.transformer.")+ return {k: v for k, v in clip_keys.items() if not k.startswith("resizer.")}++ def process_unet_state_dict(self, state_dict):+ self._clip_stash = {k: state_dict.pop(k) for k in list(state_dict.keys()) if "language_backbone" in k and "resizer" not in k}+ # SAM3.1: remap tracker.model.* -> tracker.*+ for k in list(state_dict.keys()):+ if k.startswith("tracker.model."):+ state_dict["tracker." + k[len("tracker.model."):]] = state_dict.pop(k)+ # SAM3.1: remove per-block freqs_cis buffers (computed dynamically)+ for k in [k for k in list(state_dict.keys()) if ".attn.freqs_cis" in k]:+ state_dict.pop(k)+ # Split fused QKV projections+ for k in [k for k in list(state_dict.keys()) if k.endswith((".in_proj_weight", ".in_proj_bias"))]:+ t = state_dict.pop(k)+ base, suffix = k.rsplit(".in_proj_", 1)+ s = ".weight" if suffix == "weight" else ".bias"+ d = t.shape[0] // 3+ state_dict[base + ".q_proj" + s] = t[:d]+ state_dict[base + ".k_proj" + s] = t[d:2*d]+ state_dict[base + ".v_proj" + s] = t[2*d:]+ # Remap tracker SAM decoder transformer key names to match sam.py TwoWayTransformer+ for k in list(state_dict.keys()):+ if "sam_mask_decoder.transformer." not in k:+ continue+ new_k = k.replace(".mlp.lin1.", ".mlp.0.").replace(".mlp.lin2.", ".mlp.2.").replace(".norm_final_attn.", ".norm_final.")+ if new_k != k:+ state_dict[new_k] = state_dict.pop(k)+ return state_dict++ def get_model(self, state_dict, prefix="", device=None):+ return model_base.SAM3(self, device=device)++ def clip_target(self, state_dict={}):+ import comfy.text_encoders.sam3_clip+ return supported_models_base.ClipTarget(comfy.text_encoders.sam3_clip.SAM3TokenizerWrapper, comfy.text_encoders.sam3_clip.SAM3ClipModelWrapper)+++class SAM31(SAM3):+ unet_config = {"image_model": "SAM31"}+++models = [LotusD, Stable_Zero123, SD15_instructpix2pix, SD15, SD20, SD21UnclipL, SD21UnclipH, SDXL_instructpix2pix, SDXLRefiner, SDXL, SSD1B, KOALA_700M, KOALA_1B, Segmind_Vega, SD_X4Upscaler, Stable_Cascade_C, Stable_Cascade_B, SV3D_u, SV3D_p, SD3, StableAudio, AuraFlow, PixArtAlpha, PixArtSigma, HunyuanDiT, HunyuanDiT1, FluxInpaint, Flux, LongCatImage, FluxSchnell, GenmoMochi, LTXV, LTXAV, HunyuanVideo15_SR_Distilled, HunyuanVideo15, HunyuanImage21Refiner, HunyuanImage21, HunyuanVideoSkyreelsI2V, HunyuanVideoI2V, HunyuanVideo, CosmosT2V, CosmosI2V, CosmosT2IPredict2, CosmosI2VPredict2, ZImagePixelSpace, ZImage, Lumina2, WAN22_T2V, WAN21_T2V, WAN21_I2V, WAN21_FunControl2V, WAN21_Vace, WAN21_Camera, WAN22_Camera, WAN22_S2V, WAN21_HuMo, WAN22_Animate, WAN21_FlowRVS, WAN21_SCAIL, Hunyuan3Dv2mini, Hunyuan3Dv2, Hunyuan3Dv2_1, HiDream, Chroma, ChromaRadiance, ACEStep, ACEStep15, Omnigen2, QwenImage, Flux2, Kandinsky5Image, Kandinsky5, Anima, RT_DETR_v4, ErnieImage, SAM3, SAM31]models += [SVD_img2vid]diff --git a/comfy/text_encoders/sam3_clip.py b/comfy/text_encoders/sam3_clip.pynew file mode 100644index 000000000..11cb7d9db--- /dev/null+++ b/comfy/text_encoders/sam3_clip.py@@ -0,0 +1,97 @@+import re+from comfy import sd1_clip++SAM3_CLIP_CONFIG = {+ "architectures": ["CLIPTextModel"],+ "hidden_act": "quick_gelu",+ "hidden_size": 1024,+ "intermediate_size": 4096,+ "num_attention_heads": 16,+ "num_hidden_layers": 24,+ "max_position_embeddings": 32,+ "projection_dim": 512,+ "vocab_size": 49408,+ "layer_norm_eps": 1e-5,+ "eos_token_id": 49407,+}+++class SAM3ClipModel(sd1_clip.SDClipModel):+ def __init__(self, device="cpu", dtype=None, model_options={}):+ super().__init__(device=device, dtype=dtype, max_length=32, layer="last", textmodel_json_config=SAM3_CLIP_CONFIG, special_tokens={"start": 49406, "end": 49407, "pad": 0}, return_projected_pooled=False, return_attention_masks=True, enable_attention_masks=True, model_options=model_options)+++class SAM3Tokenizer(sd1_clip.SDTokenizer):+ def __init__(self, embedding_directory=None, tokenizer_data={}):+ super().__init__(max_length=32, pad_with_end=False, pad_token=0, embedding_directory=embedding_directory, embedding_size=1024, embedding_key="sam3_clip", tokenizer_data=tokenizer_data)+ self.disable_weights = True+++def _parse_prompts(text):+ """Split comma-separated prompts with optional :N max detections per category"""+ text = text.replace("(", "").replace(")", "")+ parts = [p.strip() for p in text.split(",") if p.strip()]+ result = []+ for part in parts:+ m = re.match(r'^(.+?)\s*:\s*([\d.]+)\s*$', part)+ if m:+ text_part = m.group(1).strip()+ val = m.group(2)+ max_det = max(1, round(float(val)))+ result.append((text_part, max_det))+ else:+ result.append((part, 1))+ return result+++class SAM3TokenizerWrapper(sd1_clip.SD1Tokenizer):+ def __init__(self, embedding_directory=None, tokenizer_data={}):+ super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, clip_name="l", tokenizer=SAM3Tokenizer, name="sam3_clip")++ def tokenize_with_weights(self, text: str, return_word_ids=False, **kwargs):+ parsed = _parse_prompts(text)+ if len(parsed) <= 1 and (not parsed or parsed[0][1] == 1):+ return super().tokenize_with_weights(text, return_word_ids, **kwargs)+ # Tokenize each prompt part separately, store per-part batches and metadata+ inner = getattr(self, self.clip)+ per_prompt = []+ for prompt_text, max_det in parsed:+ batches = inner.tokenize_with_weights(prompt_text, return_word_ids, **kwargs)+ per_prompt.append((batches, max_det))+ # Main output uses first prompt's tokens (for compatibility)+ out = {self.clip_name: per_prompt[0][0], "sam3_per_prompt": per_prompt}+ return out+++class SAM3ClipModelWrapper(sd1_clip.SD1ClipModel):+ def __init__(self, device="cpu", dtype=None, model_options={}, **kwargs):+ super().__init__(device=device, dtype=dtype, model_options=model_options, clip_name="l", clip_model=SAM3ClipModel, name="sam3_clip")++ def encode_token_weights(self, token_weight_pairs):+ per_prompt = token_weight_pairs.pop("sam3_per_prompt", None)+ if per_prompt is None:+ return super().encode_token_weights(token_weight_pairs)++ # Encode each prompt separately, pack into extra dict+ inner = getattr(self, self.clip)+ multi_cond = []+ first_pooled = None+ for batches, max_det in per_prompt:+ out = inner.encode_token_weights(batches)+ cond, pooled = out[0], out[1]+ extra = out[2] if len(out) > 2 else {}+ if first_pooled is None:+ first_pooled = pooled+ multi_cond.append({+ "cond": cond,+ "attention_mask": extra.get("attention_mask"),+ "max_detections": max_det,+ })++ # Return first prompt as main (for non-SAM3 consumers), all prompts in metadata+ main = multi_cond[0]+ main_extra = {}+ if main["attention_mask"] is not None:+ main_extra["attention_mask"] = main["attention_mask"]+ main_extra["sam3_multi_cond"] = multi_cond+ return (main["cond"], first_pooled, main_extra)diff --git a/comfy_extras/nodes_sam3.py b/comfy_extras/nodes_sam3.pynew file mode 100644index 000000000..5cf92ccb3--- /dev/null+++ b/comfy_extras/nodes_sam3.py@@ -0,0 +1,529 @@+"""+SAM3 (Segment Anything 3) nodes for detection, segmentation, and video tracking.+"""++from typing_extensions import override++import json+import os+import torch+import torch.nn.functional as F+import comfy.model_management+import comfy.utils+import folder_paths+from comfy_api.latest import ComfyExtension, io, ui+import av+from fractions import Fraction+++def _extract_text_prompts(conditioning, device, dtype):+ """Extract list of (text_embeddings, text_mask) from conditioning."""+ cond_meta = conditioning[0][1]+ multi = cond_meta.get("sam3_multi_cond")+ prompts = []+ if multi is not None:+ for entry in multi:+ emb = entry["cond"].to(device=device, dtype=dtype)+ mask = entry["attention_mask"].to(device) if entry["attention_mask"] is not None else None+ if mask is None:+ mask = torch.ones(emb.shape[0], emb.shape[1], dtype=torch.int64, device=device)+ prompts.append((emb, mask, entry.get("max_detections", 1)))+ else:+ emb = conditioning[0][0].to(device=device, dtype=dtype)+ mask = cond_meta.get("attention_mask")+ if mask is not None:+ mask = mask.to(device)+ else:+ mask = torch.ones(emb.shape[0], emb.shape[1], dtype=torch.int64, device=device)+ prompts.append((emb, mask, 1))+ return prompts+++def _refine_mask(sam3_model, orig_image_hwc, coarse_mask, box_xyxy, H, W, device, dtype, iterations):+ """Refine a coarse detector mask via SAM decoder, cropping to the detection box.++ Returns: [1, H, W] binary mask+ """+ def _coarse_fallback():+ return (F.interpolate(coarse_mask.unsqueeze(0).unsqueeze(0), size=(H, W),+ mode="bilinear", align_corners=False)[0] > 0).float()++ if iterations <= 0:+ return _coarse_fallback()++ pad_frac = 0.1+ x1, y1, x2, y2 = box_xyxy.tolist()+ bw, bh = x2 - x1, y2 - y1+ cx1 = max(0, int(x1 - bw * pad_frac))+ cy1 = max(0, int(y1 - bh * pad_frac))+ cx2 = min(W, int(x2 + bw * pad_frac))+ cy2 = min(H, int(y2 + bh * pad_frac))+ if cx2 <= cx1 or cy2 <= cy1:+ return _coarse_fallback()++ crop = orig_image_hwc[cy1:cy2, cx1:cx2, :3]+ crop_1008 = comfy.utils.common_upscale(crop.unsqueeze(0).movedim(-1, 1), 1008, 1008, "bilinear", crop="disabled")+ crop_frame = crop_1008.to(device=device, dtype=dtype)+ crop_h, crop_w = cy2 - cy1, cx2 - cx1++ # Crop coarse mask and refine via SAM on the cropped image+ mask_h, mask_w = coarse_mask.shape[-2:]+ mx1, my1 = int(cx1 / W * mask_w), int(cy1 / H * mask_h)+ mx2, my2 = int(cx2 / W * mask_w), int(cy2 / H * mask_h)+ if mx2 <= mx1 or my2 <= my1:+ return _coarse_fallback()+ mask_logit = coarse_mask[..., my1:my2, mx1:mx2].unsqueeze(0).unsqueeze(0)+ for _ in range(iterations):+ coarse_input = F.interpolate(mask_logit, size=(1008, 1008), mode="bilinear", align_corners=False)+ mask_logit = sam3_model.forward_segment(crop_frame, mask_inputs=coarse_input)++ refined_crop = F.interpolate(mask_logit, size=(crop_h, crop_w), mode="bilinear", align_corners=False)+ full_mask = torch.zeros(1, 1, H, W, device=device, dtype=dtype)+ full_mask[:, :, cy1:cy2, cx1:cx2] = refined_crop+ coarse_full = F.interpolate(coarse_mask.unsqueeze(0).unsqueeze(0), size=(H, W), mode="bilinear", align_corners=False)+ return ((full_mask[0] > 0) | (coarse_full[0] > 0)).float()++++class SAM3_Detect(io.ComfyNode):+ """Open-vocabulary detection and segmentation using text, box, or point prompts."""++ @classmethod+ def define_schema(cls):+ return io.Schema(+ node_id="SAM3_Detect",+ display_name="SAM3 Detect",+ category="detection/",+ search_aliases=["sam3", "segment anything", "open vocabulary", "text detection", "segment"],+ inputs=[+ io.Model.Input("model", display_name="model"),+ io.Image.Input("image", display_name="image"),+ io.Conditioning.Input("conditioning", display_name="conditioning", optional=True, tooltip="Text conditioning from CLIPTextEncode"),+ io.BoundingBox.Input("bboxes", display_name="bboxes", force_input=True, optional=True, tooltip="Bounding boxes to segment within"),+ io.String.Input("positive_coords", display_name="positive_coords", force_input=True, optional=True, tooltip="Positive point prompts as JSON [{\"x\": int, \"y\": int}, ...] (pixel coords)"),+ io.String.Input("negative_coords", display_name="negative_coords", force_input=True, optional=True, tooltip="Negative point prompts as JSON [{\"x\": int, \"y\": int}, ...] (pixel coords)"),+ io.Float.Input("threshold", display_name="threshold", default=0.5, min=0.0, max=1.0, step=0.01),+ io.Int.Input("refine_iterations", display_name="refine_iterations", default=2, min=0, max=5, tooltip="SAM decoder refinement passes (0=use raw detector masks)"),+ io.Boolean.Input("individual_masks", display_name="individual_masks", default=False, tooltip="Output per-object masks instead of union"),+ ],+ outputs=[+ io.Mask.Output("masks"),+ io.BoundingBox.Output("bboxes"),+ ],+ )++ @classmethod+ def execute(cls, model, image, conditioning=None, bboxes=None, positive_coords=None, negative_coords=None, threshold=0.5, refine_iterations=2, individual_masks=False) -> io.NodeOutput:+ B, H, W, C = image.shape+ image_in = comfy.utils.common_upscale(image[..., :3].movedim(-1, 1), 1008, 1008, "bilinear", crop="disabled")++ # Convert bboxes to normalized cxcywh format, per-frame list of [1, N, 4] tensors.+ # Supports: single dict (all frames), list[dict] (all frames), list[list[dict]] (per-frame).+ def _boxes_to_tensor(box_list):+ coords = []+ for d in box_list:+ cx = (d["x"] + d["width"] / 2) / W+ cy = (d["y"] + d["height"] / 2) / H+ coords.append([cx, cy, d["width"] / W, d["height"] / H])+ return torch.tensor([coords], dtype=torch.float32) # [1, N, 4]++ per_frame_boxes = None+ if bboxes is not None:+ if isinstance(bboxes, dict):+ # Single box → same for all frames+ shared = _boxes_to_tensor([bboxes])+ per_frame_boxes = [shared] * B+ elif isinstance(bboxes, list) and len(bboxes) > 0 and isinstance(bboxes[0], list):+ # list[list[dict]] → per-frame boxes+ per_frame_boxes = [_boxes_to_tensor(frame_boxes) if frame_boxes else None for frame_boxes in bboxes]+ # Pad to B if fewer frames provided+ while len(per_frame_boxes) < B:+ per_frame_boxes.append(per_frame_boxes[-1] if per_frame_boxes else None)+ elif isinstance(bboxes, list) and len(bboxes) > 0:+ # list[dict] → same boxes for all frames+ shared = _boxes_to_tensor(bboxes)+ per_frame_boxes = [shared] * B++ # Parse point prompts from JSON (KJNodes PointsEditor format: [{"x": int, "y": int}, ...])+ pos_pts = json.loads(positive_coords) if positive_coords else []+ neg_pts = json.loads(negative_coords) if negative_coords else []+ has_points = len(pos_pts) > 0 or len(neg_pts) > 0++ comfy.model_management.load_model_gpu(model)+ device = comfy.model_management.get_torch_device()+ dtype = model.model.get_dtype()+ sam3_model = model.model.diffusion_model++ # Build point inputs for tracker SAM decoder path+ point_inputs = None+ if has_points:+ all_coords = [[p["x"] / W * 1008, p["y"] / H * 1008] for p in pos_pts] + \+ [[p["x"] / W * 1008, p["y"] / H * 1008] for p in neg_pts]+ all_labels = [1] * len(pos_pts) + [0] * len(neg_pts)+ point_inputs = {+ "point_coords": torch.tensor([all_coords], dtype=dtype, device=device),+ "point_labels": torch.tensor([all_labels], dtype=torch.int32, device=device),+ }++ cond_list = _extract_text_prompts(conditioning, device, dtype) if conditioning is not None and len(conditioning) > 0 else []+ has_text = len(cond_list) > 0++ # Run per-image through detector (text/boxes) and/or tracker (points)+ all_bbox_dicts = []+ all_masks = []+ pbar = comfy.utils.ProgressBar(B)++ for b in range(B):+ frame = image_in[b:b+1].to(device=device, dtype=dtype)+ b_boxes = None+ if per_frame_boxes is not None and per_frame_boxes[b] is not None:+ b_boxes = per_frame_boxes[b].to(device=device, dtype=dtype)++ frame_bbox_dicts = []+ frame_masks = []++ # Point prompts: tracker SAM decoder path with iterative refinement+ if point_inputs is not None:+ mask_logit = sam3_model.forward_segment(frame, point_inputs=point_inputs)+ for _ in range(max(0, refine_iterations - 1)):+ mask_logit = sam3_model.forward_segment(frame, mask_inputs=mask_logit)+ mask = F.interpolate(mask_logit, size=(H, W), mode="bilinear", align_corners=False)+ frame_masks.append((mask[0] > 0).float())++ # Box prompts: SAM decoder path (segment inside each box)+ if b_boxes is not None and not has_text:+ for box_cxcywh in b_boxes[0]:+ cx, cy, bw, bh = box_cxcywh.tolist()+ # Convert cxcywh normalized → xyxy in 1008 space → [1, 2, 2] corners+ sam_box = torch.tensor([[[(cx - bw/2) * 1008, (cy - bh/2) * 1008],+ [(cx + bw/2) * 1008, (cy + bh/2) * 1008]]],+ device=device, dtype=dtype)+ mask_logit = sam3_model.forward_segment(frame, box_inputs=sam_box)+ for _ in range(max(0, refine_iterations - 1)):+ mask_logit = sam3_model.forward_segment(frame, mask_inputs=mask_logit)+ mask = F.interpolate(mask_logit, size=(H, W), mode="bilinear", align_corners=False)+ frame_masks.append((mask[0] > 0).float())++ # Text prompts: run detector per text prompt (each detects one category)+ for text_embeddings, text_mask, max_det in cond_list:+ results = sam3_model(+ frame, text_embeddings=text_embeddings, text_mask=text_mask,+ boxes=b_boxes, threshold=threshold, orig_size=(H, W))++ pred_boxes = results["boxes"][0]+ scores = results["scores"][0]+ masks = results["masks"][0]++ probs = scores.sigmoid()+ keep = probs > threshold+ kept_boxes = pred_boxes[keep].cpu()+ kept_scores = probs[keep].cpu()+ kept_masks = masks[keep]++ order = kept_scores.argsort(descending=True)[:max_det]+ kept_boxes = kept_boxes[order]+ kept_scores = kept_scores[order]+ kept_masks = kept_masks[order]++ for box, score in zip(kept_boxes, kept_scores):+ frame_bbox_dicts.append({+ "x": float(box[0]), "y": float(box[1]),+ "width": float(box[2] - box[0]), "height": float(box[3] - box[1]),+ "score": float(score),+ })+ for m, box in zip(kept_masks, kept_boxes):+ frame_masks.append(_refine_mask(+ sam3_model, image[b], m, box, H, W, device, dtype, refine_iterations))++ all_bbox_dicts.append(frame_bbox_dicts)+ if len(frame_masks) > 0:+ combined = torch.cat(frame_masks, dim=0) # [N_obj, H, W]+ if individual_masks:+ all_masks.append(combined)+ else:+ all_masks.append((combined > 0).any(dim=0).float())+ else:+ if individual_masks:+ all_masks.append(torch.zeros(0, H, W, device=comfy.model_management.intermediate_device()))+ else:+ all_masks.append(torch.zeros(H, W, device=comfy.model_management.intermediate_device()))+ pbar.update(1)++ idev = comfy.model_management.intermediate_device()+ all_masks = [m.to(idev) for m in all_masks]+ mask_out = torch.cat(all_masks, dim=0) if individual_masks else torch.stack(all_masks)+ return io.NodeOutput(mask_out, all_bbox_dicts)+++SAM3TrackData = io.Custom("SAM3_TRACK_DATA")++class SAM3_VideoTrack(io.ComfyNode):+ """Track objects across video frames using SAM3's memory-based tracker."""++ @classmethod+ def define_schema(cls):+ return io.Schema(+ node_id="SAM3_VideoTrack",+ display_name="SAM3 Video Track",+ category="detection/",+ search_aliases=["sam3", "video", "track", "propagate"],+ inputs=[+ io.Image.Input("images", display_name="images", tooltip="Video frames as batched images"),+ io.Model.Input("model", display_name="model"),+ io.Mask.Input("initial_mask", display_name="initial_mask", optional=True, tooltip="Mask(s) for the first frame to track (one per object)"),+ io.Conditioning.Input("conditioning", display_name="conditioning", optional=True, tooltip="Text conditioning for detecting new objects during tracking"),+ io.Float.Input("detection_threshold", display_name="detection_threshold", default=0.5, min=0.0, max=1.0, step=0.01, tooltip="Score threshold for text-prompted detection"),+ io.Int.Input("max_objects", display_name="max_objects", default=0, min=0, tooltip="Max tracked objects (0=unlimited). Initial masks count toward this limit."),+ io.Int.Input("detect_interval", display_name="detect_interval", default=1, min=1, tooltip="Run detection every N frames (1=every frame). Higher values save compute."),+ ],+ outputs=[+ SAM3TrackData.Output("track_data", display_name="track_data"),+ ],+ )++ @classmethod+ def execute(cls, images, model, initial_mask=None, conditioning=None, detection_threshold=0.5, max_objects=0, detect_interval=1) -> io.NodeOutput:+ N, H, W, C = images.shape++ comfy.model_management.load_model_gpu(model)+ device = comfy.model_management.get_torch_device()+ dtype = model.model.get_dtype()+ sam3_model = model.model.diffusion_model++ frames = images[..., :3].movedim(-1, 1)+ frames_in = comfy.utils.common_upscale(frames, 1008, 1008, "bilinear", crop="disabled").to(device=device, dtype=dtype)++ init_masks = None+ if initial_mask is not None:+ init_masks = initial_mask.unsqueeze(1).to(device=device, dtype=dtype)++ pbar = comfy.utils.ProgressBar(N)++ text_prompts = None+ if conditioning is not None and len(conditioning) > 0:+ text_prompts = [(emb, mask) for emb, mask, _ in _extract_text_prompts(conditioning, device, dtype)]+ elif initial_mask is None:+ raise ValueError("Either initial_mask or conditioning must be provided")++ result = sam3_model.forward_video(+ images=frames_in, initial_masks=init_masks, pbar=pbar, text_prompts=text_prompts,+ new_det_thresh=detection_threshold, max_objects=max_objects,+ detect_interval=detect_interval)+ result["orig_size"] = (H, W)+ return io.NodeOutput(result)+++class SAM3_TrackPreview(io.ComfyNode):+ """Visualize tracked objects with distinct colors as a video preview. No tensor output — saves to temp video."""++ @classmethod+ def define_schema(cls):+ return io.Schema(+ node_id="SAM3_TrackPreview",+ display_name="SAM3 Track Preview",+ category="detection/",+ inputs=[+ SAM3TrackData.Input("track_data", display_name="track_data"),+ io.Image.Input("images", display_name="images", optional=True),+ io.Float.Input("opacity", display_name="opacity", default=0.5, min=0.0, max=1.0, step=0.05),+ io.Float.Input("fps", display_name="fps", default=24.0, min=1.0, max=120.0, step=1.0),+ ],+ is_output_node=True,+ )++ COLORS = [+ (0.12, 0.47, 0.71), (1.0, 0.5, 0.05), (0.17, 0.63, 0.17), (0.84, 0.15, 0.16),+ (0.58, 0.4, 0.74), (0.55, 0.34, 0.29), (0.89, 0.47, 0.76), (0.5, 0.5, 0.5),+ (0.74, 0.74, 0.13), (0.09, 0.75, 0.81), (0.94, 0.76, 0.06), (0.42, 0.68, 0.84),+ ]++ # 5x3 bitmap font atlas for digits 0-9 [10, 5, 3]+ _glyph_cache = {} # (device, scale) -> (glyphs, outlines, gh, gw, oh, ow)++ @staticmethod+ def _get_glyphs(device, scale=3):+ key = (device, scale)+ if key in SAM3_TrackPreview._glyph_cache:+ return SAM3_TrackPreview._glyph_cache[key]+ atlas = torch.tensor([+ [[1,1,1],[1,0,1],[1,0,1],[1,0,1],[1,1,1]],+ [[0,1,0],[1,1,0],[0,1,0],[0,1,0],[1,1,1]],+ [[1,1,1],[0,0,1],[1,1,1],[1,0,0],[1,1,1]],+ [[1,1,1],[0,0,1],[1,1,1],[0,0,1],[1,1,1]],+ [[1,0,1],[1,0,1],[1,1,1],[0,0,1],[0,0,1]],+ [[1,1,1],[1,0,0],[1,1,1],[0,0,1],[1,1,1]],+ [[1,1,1],[1,0,0],[1,1,1],[1,0,1],[1,1,1]],+ [[1,1,1],[0,0,1],[0,0,1],[0,0,1],[0,0,1]],+ [[1,1,1],[1,0,1],[1,1,1],[1,0,1],[1,1,1]],+ [[1,1,1],[1,0,1],[1,1,1],[0,0,1],[1,1,1]],+ ], dtype=torch.bool)+ glyphs, outlines = [], []+ for d in range(10):+ g = atlas[d].repeat_interleave(scale, 0).repeat_interleave(scale, 1)+ padded = F.pad(g.float().unsqueeze(0).unsqueeze(0), (1,1,1,1))+ o = (F.max_pool2d(padded, 3, stride=1, padding=1)[0, 0] > 0)+ glyphs.append(g.to(device))+ outlines.append(o.to(device))+ gh, gw = glyphs[0].shape+ oh, ow = outlines[0].shape+ SAM3_TrackPreview._glyph_cache[key] = (glyphs, outlines, gh, gw, oh, ow)+ return SAM3_TrackPreview._glyph_cache[key]++ @staticmethod+ def _draw_number_gpu(frame, number, cx, cy, color, scale=3):+ """Draw a number on a GPU tensor [H, W, 3] float 0-1 at (cx, cy) with outline."""+ H, W = frame.shape[:2]+ device = frame.device+ glyphs, outlines, gh, gw, oh, ow = SAM3_TrackPreview._get_glyphs(device, scale)+ color_t = torch.tensor(color, device=device, dtype=frame.dtype)+ digs = [int(d) for d in str(number)]+ total_w = len(digs) * (gw + scale) - scale+ x0 = cx - total_w // 2+ y0 = cy - gh // 2+ for i, d in enumerate(digs):+ dx = x0 + i * (gw + scale)+ # Black outline+ oy0, ox0 = y0 - 1, dx - 1+ osy1, osx1 = max(0, -oy0), max(0, -ox0)+ osy2, osx2 = min(oh, H - oy0), min(ow, W - ox0)+ if osy2 > osy1 and osx2 > osx1:+ fy1, fx1 = oy0 + osy1, ox0 + osx1+ frame[fy1:fy1+(osy2-osy1), fx1:fx1+(osx2-osx1)][outlines[d][osy1:osy2, osx1:osx2]] = 0+ # Colored fill+ sy1, sx1 = max(0, -y0), max(0, -dx)+ sy2, sx2 = min(gh, H - y0), min(gw, W - dx)+ if sy2 > sy1 and sx2 > sx1:+ fy1, fx1 = y0 + sy1, dx + sx1+ frame[fy1:fy1+(sy2-sy1), fx1:fx1+(sx2-sx1)][glyphs[d][sy1:sy2, sx1:sx2]] = color_t++ @classmethod+ def execute(cls, track_data, images=None, opacity=0.5, fps=24.0) -> io.NodeOutput:++ from comfy.ldm.sam3.tracker import unpack_masks+ packed = track_data["packed_masks"]+ H, W = track_data["orig_size"]+ if images is not None:+ H, W = images.shape[1], images.shape[2]+ if packed is None:+ N, N_obj = track_data["n_frames"], 0+ else:+ N, N_obj = packed.shape[0], packed.shape[1]++ import uuid+ gpu = comfy.model_management.get_torch_device()+ temp_dir = folder_paths.get_temp_directory()+ filename = f"sam3_track_preview_{uuid.uuid4().hex[:8]}.mp4"+ filepath = os.path.join(temp_dir, filename)+ with av.open(filepath, mode='w') as output:+ stream = output.add_stream('h264', rate=Fraction(round(fps * 1000), 1000))+ stream.width = W+ stream.height = H+ stream.pix_fmt = 'yuv420p'++ frame_cpu = torch.empty(H, W, 3, dtype=torch.uint8)+ frame_np = frame_cpu.numpy()+ if N_obj > 0:+ colors_t = torch.tensor([cls.COLORS[i % len(cls.COLORS)] for i in range(N_obj)],+ device=gpu, dtype=torch.float32)+ grid_y = torch.arange(H, device=gpu).view(1, H, 1)+ grid_x = torch.arange(W, device=gpu).view(1, 1, W)+ for t in range(N):+ if images is not None and t < images.shape[0]:+ frame = images[t].clone()+ else:+ frame = torch.zeros(H, W, 3)++ if N_obj > 0:+ frame_binary = unpack_masks(packed[t:t+1].to(gpu)) # [1, N_obj, H, W] bool+ frame_masks = F.interpolate(frame_binary.float(), size=(H, W), mode="nearest")[0]+ frame_gpu = frame.to(gpu)+ bool_masks = frame_masks > 0.5+ any_mask = bool_masks.any(dim=0)+ if any_mask.any():+ obj_idx_map = bool_masks.to(torch.uint8).argmax(dim=0)+ color_overlay = colors_t[obj_idx_map]+ mask_3d = any_mask.unsqueeze(-1)+ frame_gpu = torch.where(mask_3d, frame_gpu * (1 - opacity) + color_overlay * opacity, frame_gpu)+ area = bool_masks.sum(dim=(-1, -2)).clamp_(min=1)+ cy = (bool_masks * grid_y).sum(dim=(-1, -2)) // area+ cx = (bool_masks * grid_x).sum(dim=(-1, -2)) // area+ has = area > 1+ scores = track_data.get("scores", [])+ for obj_idx in range(N_obj):+ if has[obj_idx]:+ _cx, _cy = int(cx[obj_idx]), int(cy[obj_idx])+ color = cls.COLORS[obj_idx % len(cls.COLORS)]+ SAM3_TrackPreview._draw_number_gpu(frame_gpu, obj_idx, _cx, _cy, color)+ if obj_idx < len(scores) and scores[obj_idx] < 1.0:+ SAM3_TrackPreview._draw_number_gpu(frame_gpu, int(scores[obj_idx] * 100),+ _cx, _cy + 5 * 3 + 3, color, scale=2)+ frame_cpu.copy_(frame_gpu.clamp_(0, 1).mul_(255).byte())+ else:+ frame_cpu.copy_(frame.clamp_(0, 1).mul_(255).byte())++ vframe = av.VideoFrame.from_ndarray(frame_np, format='rgb24')+ output.mux(stream.encode(vframe.reformat(format='yuv420p')))+ output.mux(stream.encode(None))+ return io.NodeOutput(ui=ui.PreviewVideo([ui.SavedResult(filename, "", io.FolderType.temp)]))+++class SAM3_TrackToMask(io.ComfyNode):+ """Select tracked objects by index and output as mask."""++ @classmethod+ def define_schema(cls):+ return io.Schema(+ node_id="SAM3_TrackToMask",+ display_name="SAM3 Track to Mask",+ category="detection/",+ inputs=[+ SAM3TrackData.Input("track_data", display_name="track_data"),+ io.String.Input("object_indices", display_name="object_indices", default="",+ tooltip="Comma-separated object indices to include (e.g. '0,2,3'). Empty = all objects."),+ ],+ outputs=[+ io.Mask.Output("masks", display_name="masks"),+ ],+ )++ @classmethod+ def execute(cls, track_data, object_indices="") -> io.NodeOutput:+ from comfy.ldm.sam3.tracker import unpack_masks+ packed = track_data["packed_masks"]+ H, W = track_data["orig_size"]++ if packed is None:+ N = track_data["n_frames"]+ return io.NodeOutput(torch.zeros(N, H, W, device=comfy.model_management.intermediate_device()))++ N, N_obj = packed.shape[0], packed.shape[1]++ if object_indices.strip():+ indices = [int(i.strip()) for i in object_indices.split(",") if i.strip().isdigit()]+ indices = [i for i in indices if 0 <= i < N_obj]+ else:+ indices = list(range(N_obj))++ if not indices:+ return io.NodeOutput(torch.zeros(N, H, W, device=comfy.model_management.intermediate_device()))++ selected = packed[:, indices]+ binary = unpack_masks(selected) # [N, len(indices), Hm, Wm] bool+ union = binary.any(dim=1, keepdim=True).float()+ mask_out = F.interpolate(union, size=(H, W), mode="bilinear", align_corners=False)[:, 0]+ return io.NodeOutput(mask_out)+++class SAM3Extension(ComfyExtension):+ @override+ async def get_node_list(self) -> list[type[io.ComfyNode]]:+ return [+ SAM3_Detect,+ SAM3_VideoTrack,+ SAM3_TrackPreview,+ SAM3_TrackToMask,+ ]+++async def comfy_entrypoint() -> SAM3Extension:+ return SAM3Extension()diff --git a/nodes.py b/nodes.pyindex bb38e07b8..fb83da896 100644--- a/nodes.py+++ b/nodes.py@@ -2459,6 +2459,7 @@ async def init_builtin_extra_nodes():"nodes_curve.py","nodes_rtdetr.py","nodes_frame_interpolation.py",+ "nodes_sam3.py"]import_failed = []