The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
feat: Pixal3D multiview model support (CORE-421) (#16048)
Jukka Seppänen488e8f8ab845
diff --git a/comfy/ldm/trellis2/model.py b/comfy/ldm/trellis2/model.pyindex 042e94aa0..fda7dfd37 100644--- a/comfy/ldm/trellis2/model.py+++ b/comfy/ldm/trellis2/model.py@@ -768,10 +768,7 @@ def _coords_to_proj_world(coords: torch.Tensor, resolution: int, mesh_scale: tornorm = coords[:, 1:].to(torch.float32) / (resolution - 1) * 2.0 - 1.0R = _PROJ_GRID_ROTATION.to(device=coords.device, dtype=torch.float32)rotated = norm @ R.T- if mesh_scale.ndim == 0:- scale_per_voxel = mesh_scale.expand(coords.shape[0])- else:- scale_per_voxel = mesh_scale.to(coords.device)[batch_ids]+ scale_per_voxel = mesh_scale.to(coords.device).reshape(-1)[batch_ids % mesh_scale.numel()]world = rotated / scale_per_voxel.unsqueeze(-1) / 2.0return world, batch_ids@@ -800,20 +797,22 @@ def _back_project_to_tokens() -> torch.Tensor:if coords_world.dim() == 2:assert batch_ids is not None- B = transform_matrix.shape[0]+ n_cond = transform_matrix.shape[0]out = torch.zeros((coords_world.shape[0], feature_map.shape[1]),device=feature_map.device, dtype=feature_map.dtype)- for b in range(B):+ # multi-seed: latent sample b uses conditioning b % n_cond+ for b in range(int(batch_ids.max().item()) + 1):mask = batch_ids == bif not mask.any():continue+ c = b % n_condp = coords_world[mask].unsqueeze(0)uv, _, _ = _project_points_to_image(- p, transform_matrix[b:b+1], camera_angle_x[b:b+1], image_resolution)+ p, transform_matrix[c:c+1], camera_angle_x[c:c+1], image_resolution)uv_ndc = (uv + 0.5) / image_resolution * 2.0 - 1.0# padding_mode='border' is load-bearing: masking out-of-frame voxels confuses# the SS DiT (~half the voxels go to zero, producing low poly + rotation drift).- sampled = _sample_features(feature_map[b:b+1], uv_ndc)+ sampled = _sample_features(feature_map[c:c+1], uv_ndc)sampled = sampled.squeeze(0).transpose(0, 1)out[mask] = sampledreturn out@@ -846,25 +845,14 @@ def compute_stage_proj_feats(batch_size: Optional[int] = None,device=None,) -> torch.Tensor:- """Back-project a Pixal3D stage's feature maps onto its target voxel/grid coords.-- For sparse (shape / texture) stages: pass ``coords`` (with ``coord_resolution``).- Returns ``[N_voxels, C]`` per-voxel features with channel count =- LR channels + optional HR channels.-- For the dense SS stage: pass ``dense_grid_resolution`` (16) + ``batch_size``.- Returns ``[B, R^3, C]`` features for the dense grid.-- """+ """Back-project a stage's feature maps onto sparse coords [N, C] or the dense SS grid [B, R^3, C]; views at stride num_views are averaged."""if device is None:device = coords.device if coords is not None else proj_pack["mesh_scale"].devicemesh_scale = proj_pack["mesh_scale"].to(device)T = proj_pack["transform_matrix"].to(device)cam_angle = proj_pack["camera_angle_x"].to(device)+ num_views = int(proj_pack.get("num_views", 1))feat_map_lr, feat_map_hr, image_resolution = _select_stage_entry(proj_pack, stage)- feat_map_lr = feat_map_lr.to(device)- if feat_map_hr is not None:- feat_map_hr = feat_map_hr.to(device)if coords is not None:if coord_resolution is None:@@ -877,13 +865,18 @@ def compute_stage_proj_feats(device=device, dtype=torch.float32)batch_ids = None- proj_lr = _back_project_to_tokens(coords_world, feat_map_lr, T, cam_angle,+ out = None+ for v in range(num_views):+ sel = slice(v, None, num_views)+ proj = _back_project_to_tokens(coords_world, feat_map_lr[sel].to(device), T[sel], cam_angle[sel],image_resolution=image_resolution, batch_ids=batch_ids)- if feat_map_hr is not None:- proj_hr = _back_project_to_tokens(coords_world, feat_map_hr, T, cam_angle,- image_resolution=image_resolution, batch_ids=batch_ids)- return torch.cat([proj_lr, proj_hr], dim=-1)- return proj_lr+ if feat_map_hr is not None:+ proj_hr = _back_project_to_tokens(coords_world, feat_map_hr[sel].to(device), T[sel], cam_angle[sel],+ image_resolution=image_resolution, batch_ids=batch_ids)+ proj = torch.cat([proj, proj_hr], dim=-1)+ # average views in fp32 like upstream+ out = proj.float() if out is None else out.add_(proj)+ return out.div_(num_views).to(proj.dtype)def _shape_proj_cond(global_cond: torch.Tensor, image_attn_mode: str,@@ -917,6 +910,11 @@ def _shape_proj_cond(global_cond: torch.Tensor, image_attn_mode: str,f"sub-model expects {proj_in_channels}.{hint}")+ # multi-seed: latent sample i uses conditioning i % B+ if batch_ids is None and logical_batch is not None and proj_feats.shape[0] != logical_batch:+ reps = -(-logical_batch // proj_feats.shape[0])+ proj_feats = proj_feats.repeat((reps,) + (1,) * (proj_feats.ndim - 1))[:logical_batch]+# CFG-duplicate proj_feats to match the model's eval batch.if eval_batch is not None and logical_batch is not None and eval_batch > logical_batch:repeats = eval_batch // logical_batch@@ -1125,7 +1123,7 @@ class Trellis2(nn.Module):else: # structurestruct_attn = self.image_attn_mode_structure- logical_batch_ss = proj_feats.shape[0] if proj_feats is not None else x.shape[0]+ logical_batch_ss = x.shape[0] // len(cond_or_uncond) if cond_or_uncond else x.shape[0]struct_cond = contextif struct_attn != "global":struct_cond = _shape_proj_cond(context, struct_attn, proj_feats,diff --git a/comfy_extras/nodes_trellis2.py b/comfy_extras/nodes_trellis2.pyindex 1e5f535cc..86fa6845f 100644--- a/comfy_extras/nodes_trellis2.py+++ b/comfy_extras/nodes_trellis2.py@@ -7,6 +7,7 @@ from comfy_extras.nodes_mesh_postprocess import pack_variable_mesh_batchimport comfy.latent_formatsimport comfy.model_managementimport comfy.utils+import loggingimport mathimport torch@@ -690,6 +691,73 @@ def _dino_encode_batch(clip_vision_model, image, out_device, *, want_patches=Falout["composites"] = composite_listreturn out+def _naf_upsample(naf_model, lr_feat, composites, image_size, naf_target, out_device, compute_device):+ """NAF-upsample each item's DINO patch grid to naf_target, guided by its composite."""+ if naf_model is None:+ return None+ comfy.model_management.load_model_gpu(naf_model)+ inner = naf_model.model+ model_dtype = next(inner.parameters()).dtype+ out = torch.empty((len(composites), lr_feat.shape[1], *naf_target), device=out_device, dtype=model_dtype)+ for i, c in enumerate(composites):+ img_i = comfy.utils.common_upscale(c, image_size, image_size, "lanczos", "disabled").to(compute_device, model_dtype)+ lr_i = lr_feat[i:i + 1].to(compute_device, model_dtype)+ inner(img_i, lr_i, naf_target, output=out[i:i + 1])+ return out+++def _build_pixal3d_conditioning(clip_vision_model, image, transform_matrix, camera_angle_x, mesh_scale, num_views=1):+ """Per-item inputs hold B*num_views entries with each object's views consecutive; mesh_scale holds B."""+ naf_model = clip_vision_model.naf+ out_device = comfy.model_management.intermediate_device()+ compute_device = comfy.model_management.get_torch_device()++ cond = _dino_encode_batch(clip_vision_model, image, out_device, want_patches=True)+ batch_size = cond["batch_size"] // num_views+ fm_512_dino, fm_1024_dino = cond["patches_512"], cond["patches_1024"]+ composite_list = cond["composites"]++ # NAF HR targets per stage: shape_512=512, shape_1024=512, tex_1024=1024+ hr_shape_512 = _naf_upsample(naf_model, fm_512_dino, composite_list, 512, (512, 512), out_device, compute_device)+ hr_shape_1024 = _naf_upsample(naf_model, fm_1024_dino, composite_list, 1024, (512, 512), out_device, compute_device)+ hr_tex_1024 = _naf_upsample(naf_model, fm_1024_dino, composite_list, 1024, (1024, 1024), out_device, compute_device)++ # CLS + register tokens averaged over each object's views+ global_512 = cond["global_512"].unflatten(0, (batch_size, num_views)).mean(dim=1)+ global_1024 = cond["global_1024"].unflatten(0, (batch_size, num_views)).mean(dim=1)++ proj_pack = {+ "stages": {+ "ss": {"feature_map": fm_512_dino, "feature_map_hr": None, "image_resolution": 512},+ "shape_512": {"feature_map": fm_512_dino, "feature_map_hr": hr_shape_512, "image_resolution": 512},+ "shape_1024": {"feature_map": fm_1024_dino, "feature_map_hr": hr_shape_1024,"image_resolution": 1024},+ "tex_1024": {"feature_map": fm_1024_dino, "feature_map_hr": hr_tex_1024, "image_resolution": 1024},+ },+ "transform_matrix": transform_matrix.to(out_device),+ "camera_angle_x": camera_angle_x.to(out_device),+ "mesh_scale": mesh_scale.to(out_device),+ "num_views": num_views,+ "patch_size": 16,+ }++ # global_512 -> SS/shape_512 cross-attn; global_1024 -> shape_1024/tex_1024.+ ss_proj_feats = compute_stage_proj_feats(+ proj_pack, "ss", dense_grid_resolution=16, batch_size=batch_size,+ device=compute_device,+ )+ base_extras = {+ "embeds": global_1024, "proj_feat_pack": proj_pack,+ "trellis2_proj_feats": ss_proj_feats,+ }+ neg_extras = {+ "embeds": torch.zeros_like(global_1024), "proj_feat_pack": proj_pack,+ "trellis2_proj_feats": ss_proj_feats,+ }+ positive = [[global_512, base_extras]]+ negative = [[torch.zeros_like(global_512), neg_extras]]+ return IO.NodeOutput(positive, negative)++class Pixal3DConditioning(IO.ComfyNode):@classmethod@@ -715,79 +783,85 @@ class Pixal3DConditioning(IO.ComfyNode):@classmethoddef execute(cls, clip_vision_model, image, camera_angle_x) -> IO.NodeOutput:- naf_model = clip_vision_model.naf- out_device = comfy.model_management.intermediate_device()- compute_device = comfy.model_management.get_torch_device()-- cond = _dino_encode_batch(clip_vision_model, image, out_device, want_patches=True)- batch_size = cond["batch_size"]- global_512, global_1024 = cond["global_512"], cond["global_1024"]- fm_512_dino, fm_1024_dino = cond["patches_512"], cond["patches_1024"]- composite_list = cond["composites"]-- # The LR DINO grid AND the NAF HR grid are sampled separately- # NAF targets per stage: shape_512=512, shape_1024=512, tex_1024=1024.- def _naf_hr(lr_feat, composites, image_size, naf_target):- if naf_model is None or naf_target is None:- return None- comfy.model_management.load_model_gpu(naf_model)- inner = naf_model.model- model_dtype = next(inner.parameters()).dtype # set at load time (see clip_vision NAF)- hrs = []- for i, c in enumerate(composites):- img_i = comfy.utils.common_upscale(c, image_size, image_size, "lanczos", "disabled")\- .to(compute_device).to(model_dtype)- lr_i = lr_feat[i:i + 1].to(compute_device).to(model_dtype)- output = torch.empty((1, lr_i.shape[1], *naf_target), device=out_device, dtype=model_dtype)- hr_i = inner(img_i, lr_i, naf_target, output=output)- hrs.append(hr_i)- return torch.cat(hrs, dim=0)-- hr_shape_512 = _naf_hr(fm_512_dino, composite_list, 512, (512, 512))- hr_shape_1024 = _naf_hr(fm_1024_dino, composite_list, 1024, (512, 512))- hr_tex_1024 = _naf_hr(fm_1024_dino, composite_list, 1024, (1024, 1024))-+ batch_size = image.shape[0]# distance_from_fov: grid_point (-1, 0, 0) projects to pixel (0, image_resolution-1).# FOV widget is in degrees for UX; trig + downstream projection expect radians.camera_angle_x = math.radians(float(camera_angle_x))distance = 0.5 / math.tan(camera_angle_x / 2.0)- cam_angle_t = torch.tensor([camera_angle_x] * batch_size, device=out_device, dtype=torch.float32)- dist_t = torch.tensor([distance] * batch_size, device=out_device, dtype=torch.float32)- scale_t = torch.ones(batch_size, device=out_device, dtype=torch.float32)- T = build_proj_transform_matrix(dist_t, batch_size, device=out_device, dtype=torch.float32)-- proj_pack = {- "stages": {- "ss": {"feature_map": fm_512_dino, "feature_map_hr": None, "image_resolution": 512},- "shape_512": {"feature_map": fm_512_dino, "feature_map_hr": hr_shape_512, "image_resolution": 512},- "shape_1024": {"feature_map": fm_1024_dino, "feature_map_hr": hr_shape_1024,"image_resolution": 1024},- "tex_1024": {"feature_map": fm_1024_dino, "feature_map_hr": hr_tex_1024, "image_resolution": 1024},- },- "transform_matrix": T,- "camera_angle_x": cam_angle_t,- "mesh_scale": scale_t,- "distance": dist_t,- "patch_size": 16,- }+ cam_angle_t = torch.full((batch_size,), camera_angle_x)+ dist_t = torch.full((batch_size,), distance)+ T = build_proj_transform_matrix(dist_t, batch_size, dist_t.device)+ return _build_pixal3d_conditioning(clip_vision_model, image, T, cam_angle_t, torch.ones(batch_size))+++_VIEW_AZIMUTHS = {"front": 0.0, "left": 90.0, "back": 180.0, "right": 270.0}+_VIEW_PAD = 1.1 # unit cube spans 1/1.1 of the frame, upstream's example rig+- # global_512 → SS/shape_512 cross-attn; global_1024 → shape_1024/tex_1024.- ss_proj_feats = compute_stage_proj_feats(- proj_pack, "ss", dense_grid_resolution=16, batch_size=batch_size,- device=compute_device,+def _orbit_camera_to_world(azimuths_deg, elevations_deg, distance):+ """Z-up orbit cameras looking at the origin; azimuth 0 / elevation 0 is the front view."""+ az = torch.deg2rad(torch.tensor(azimuths_deg, dtype=torch.float32))+ el = torch.deg2rad(torch.tensor(elevations_deg, dtype=torch.float32))+ back = torch.stack([torch.sin(az) * torch.cos(el), -torch.cos(az) * torch.cos(el), torch.sin(el)], dim=-1)+ right = torch.stack([torch.cos(az), torch.sin(az), torch.zeros_like(az)], dim=-1)+ c2w = torch.eye(4).repeat(az.shape[0], 1, 1)+ c2w[:, :3, :3] = torch.stack([right, torch.cross(back, right, dim=-1), back], dim=-1)+ c2w[:, :3, 3] = back * distance+ return c2w+++class Pixal3DMultiViewConditioning(IO.ComfyNode):+ """Fixed orbit rig: front, left, back and right views 90 degrees apart, used as framed."""++ @classmethod+ def define_schema(cls):+ views = [IO.Image.Input(name, optional=True,+ tooltip=f"Square view of the object's {name} side, with alpha or on a black background, "+ "framed like the rig: the object spans about 1/1.1 of the frame at its widest, "+ "the same scale in every view. The first connected view (front, left, back, "+ "right order) is the front the mesh is posed to.")+ for name in _VIEW_AZIMUTHS]+ return IO.Schema(+ node_id="Pixal3DMultiViewConditioning",+ display_name="Pixal3D Multi-View Conditioning",+ category="model/conditioning/trellis2",+ inputs=[IO.ClipVision.Input("clip_vision_model", tooltip="DINOv3 ViT-L/16 ClipVision with bundled NAF weights."),+ IO.Float.Input("fov", default=20.0, min=1.0, max=170.0, step=0.01, round=False,+ tooltip="Horizontal FOV in degrees of the views as framed: 20 for rig renders and most "+ "multi-view generators, or MoGeGeometryToFOV on one of the views for photos.")]+ + views,+ outputs=[+ IO.Conditioning.Output(display_name="positive"),+ IO.Conditioning.Output(display_name="negative"),+ ],)- neg_global = torch.zeros_like(global_512)- neg_embeds = torch.zeros_like(global_1024)- base_extras = {- "embeds": global_1024, "proj_feat_pack": proj_pack,- "trellis2_proj_feats": ss_proj_feats,- }- neg_extras = {- "embeds": neg_embeds, "proj_feat_pack": proj_pack,- "trellis2_proj_feats": ss_proj_feats,- }- positive = [[global_512, base_extras]]- negative = [[neg_global, neg_extras]]- return IO.NodeOutput(positive, negative)++ @classmethod+ def execute(cls, clip_vision_model, fov, front=None, left=None, back=None, right=None) -> IO.NodeOutput:+ views = {"front": front, "left": left, "back": back, "right": right}+ names = [name for name in _VIEW_AZIMUTHS if views[name] is not None]+ if not names:+ raise ValueError("Pixal3DMultiViewConditioning needs at least one view")+ batch_size = views[names[0]].shape[0]+ num_views = len(names)+ # the first connected view is the front (upstream re-bases the rig onto view 0; on an orbit that is an azimuth shift)+ if names[0] != "front":+ logging.warning(f"Pixal3DMultiViewConditioning: no front view, the mesh will be posed with the {names[0]} view as its front")+ azimuths = [_VIEW_AZIMUTHS[name] - _VIEW_AZIMUTHS[names[0]] for name in names]+ items = []+ for b in range(batch_size):+ for name in names:+ view = views[name][b % views[name].shape[0]][None]+ if view.shape[-1] == 4:+ view = view[..., :3] * view[..., 3:4]+ if view.shape[1:3] != (1024, 1024):+ view = comfy.utils.common_upscale(view.movedim(-1, 1), 1024, 1024, "lanczos", "disabled").movedim(1, -1)+ items.append(view)+ fov = math.radians(fov)+ c2w = _orbit_camera_to_world(azimuths, [0.0] * num_views, _VIEW_PAD * 0.5 / math.tan(fov / 2.0))+ return _build_pixal3d_conditioning(clip_vision_model, torch.cat(items, dim=0), c2w.repeat(batch_size, 1, 1),+ torch.full((batch_size * num_views,), fov), torch.ones(batch_size),+ num_views=num_views)class Trellis2Extension(ComfyExtension):@@ -796,6 +870,7 @@ class Trellis2Extension(ComfyExtension):return [Trellis2Conditioning,Pixal3DConditioning,+ Pixal3DMultiViewConditioning,Trellis2ShapeStage,EmptyTrellis2LatentStructure,Trellis2TextureStage,