Skip to content

armgit / atas

Public Imported from GitHub
0

Any Time, Any Space: A World-State-Centric Framework for Agentic Video Generation

21.0 KB
Raw Blame
import torch
import torch.nn as nn
import torch.nn.functional as F
from functools import partial
from copy import deepcopy
from huggingface_hub import PyTorchModelHubMixin

from .layers.conv_head import ConvHead
from .layers.camera_head import CameraHead
from .dinov2.layers import Mlp, PatchEmbed
from .layers.attention import FlashAttentionRope
from .layers.block import BlockRope, PoseInjectBlock
from .layers.pos_embed import RoPE2D, PositionGetter
from .dinov2.hub.backbones import dinov2_vitl14, dinov2_vitl14_reg
from ..utils.geometry import se3_inverse, get_pixel, homogenize_points
from .layers.transformer_head import TransformerDecoder, ContextOnlyTransformerDecoder


class Pi3X(nn.Module, PyTorchModelHubMixin):
    def __init__(
            self,
            ckpt=None,    
            use_multimodal=True
        ):
        super().__init__()

        self.use_multimodal = use_multimodal

        # ----------------------
        #        Encoder
        # ----------------------
        self.encoder = dinov2_vitl14_reg(pretrained=False)
        self.patch_size = 14
        del self.encoder.mask_token

        # ----------------------
        #  Positonal Encoding
        # ----------------------
        freq = 100
        self.rope = RoPE2D(freq=freq)
        self.position_getter = PositionGetter()

        # ----------------------
        #        Decoder
        # ----------------------
        dec_embed_dim = 1024
        dec_num_heads = 16
        mlp_ratio = 4
        dec_depth = 36      
        self.decoder = nn.ModuleList([
            BlockRope(
                dim=dec_embed_dim,
                num_heads=dec_num_heads,
                mlp_ratio=mlp_ratio,
                qkv_bias=True,
                proj_bias=True,
                ffn_bias=True,
                drop_path=0.0,
                norm_layer=partial(nn.LayerNorm, eps=1e-6),
                act_layer=nn.GELU,
                ffn_layer=Mlp,
                init_values=0.01,
                qk_norm=True,
                attn_class=FlashAttentionRope,
                rope=self.rope
        ) for _ in range(dec_depth)])
        self.dec_embed_dim = dec_embed_dim

        num_register_tokens = 5
        self.patch_start_idx = num_register_tokens
        self.register_token = nn.Parameter(torch.randn(1, 1, num_register_tokens, self.dec_embed_dim))
        nn.init.normal_(self.register_token, std=1e-6)

        # -----------------------
        #       multi-modal
        # -----------------------
        if use_multimodal:
            ## Depth encoder
            self.depth_encoder = deepcopy(self.encoder)
            del self.depth_encoder.patch_embed
            self.depth_encoder.patch_embed = PatchEmbed(img_size=224, patch_size=14, in_chans=2, embed_dim=1024)
            self.depth_emb = nn.Parameter(torch.zeros(1, 1, 1024))

            ## Ray embedding
            self.ray_embed = PatchEmbed(img_size=224, patch_size=14, in_chans=2, embed_dim=1024)
            nn.init.constant_(self.ray_embed.proj.weight, 0)
            nn.init.constant_(self.ray_embed.proj.bias, 0)

            ## Pose inject blocks
            self.pose_inject_blk = nn.ModuleList([PoseInjectBlock(
                dim=1024,
                num_heads=16,
                mlp_ratio=4,
                qkv_bias=True,
                proj_bias=True,
                ffn_bias=True,
                drop_path=0.0,
                norm_layer=partial(nn.LayerNorm, eps=1e-6),
                act_layer=nn.GELU,
                ffn_layer=Mlp,
                init_values=0.01,
                qk_norm=True,
            ) for _ in range(5)])


        # ------------------------------
        #           Head
        # ------------------------------
        ## --------------- Point ---------------
        self.point_decoder = TransformerDecoder(
            in_dim=2*self.dec_embed_dim, 
            dec_embed_dim=1024,
            dec_num_heads=16,                # 8
            out_dim=1024,
            rope=self.rope,
        )
        # self.point_head = LinearPts3d(patch_size=14, dec_embed_dim=1024, output_dim=3)
        self.point_head = ConvHead(
                num_features=4, 
                dim_in=dec_embed_dim,
                # projects=nn.Linear(1024, 1024),
                projects=nn.Identity(),
                dim_out=[2, 1], 
                dim_proj=1024,
                dim_upsample=[256, 128, 64],
                dim_times_res_block_hidden=2,
                num_res_blocks=2,
                res_block_norm='group_norm',
                last_res_blocks=0,
                last_conv_channels=32,
                last_conv_size=1,
                using_uv=True
            )

        ## --------------- Camera ---------------
        self.camera_decoder = TransformerDecoder(
            in_dim=2*self.dec_embed_dim, 
            dec_embed_dim=1024,
            dec_num_heads=16,                # 8
            out_dim=512,
            rope=self.rope,
        )
        self.camera_head = CameraHead(dim=512)

        ## --------------- Metric ---------------
        self.metric_token = nn.Parameter(torch.randn(1, 1, 2*self.dec_embed_dim))
        self.metric_decoder = ContextOnlyTransformerDecoder(
            in_dim=2*self.dec_embed_dim, 
            dec_embed_dim=512,
            dec_num_heads=8,                # 8
            out_dim=512,
            rope=self.rope,
        )
        self.metric_head = nn.Linear(512, 1)
        nn.init.normal_(self.metric_token, std=1e-6)


        ## -------------- Conf ------------------
        self.conf_decoder = TransformerDecoder(
            in_dim=2*self.dec_embed_dim, 
            dec_embed_dim=1024,
            dec_num_heads=16,                # 8
            out_dim=1024,
            rope=self.rope,
        )
        self.conf_head = ConvHead(
            num_features=4, 
            dim_in=dec_embed_dim,
            # projects=nn.Linear(1024, 1024),
            projects=nn.Identity(),
            dim_out=[1], 
            dim_proj=1024,
            dim_upsample=[256, 128, 64],
            dim_times_res_block_hidden=2,
            num_res_blocks=2,
            res_block_norm='group_norm',
            last_res_blocks=0,
            last_conv_channels=32,
            last_conv_size=1,
            using_uv=True
        )

        # For ImageNet Normalize
        image_mean = torch.tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1)
        image_std = torch.tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1)

        self.register_buffer("image_mean", image_mean)
        self.register_buffer("image_std", image_std)


    def forward(
        self,
        imgs,
        depths=None,
        intrinsics=None,
        rays=None,
        poses=None,
        with_prior=True,
        mask_add_depth=None,
        mask_add_ray=None,
        mask_add_pose=None,
    ):
        """
        Forward pass with optional multimodal conditions.

        Args:
            imgs (torch.Tensor): Input RGB images valued in [0, 1].
                Shape: (B, N, 3, H, W).
            intrinsics (torch.Tensor, optional): Camera intrinsic matrices.
                Shape: (B, N, 3, 3).
                Values are in pixel coordinates (not normalized).
            rays (torch.Tensor, optional): Pre-computed ray directions (unit vectors).
                Shape: (B, N, H, W, 3).
                Can replace `intrinsics` as a geometric condition.
            poses (torch.Tensor, optional): Camera-to-World matrices.
                Shape: (B, N, 4, 4).
                Coordinate system: OpenCV convention (Right-Down-Forward).
            depths (torch.Tensor, optional): Ground truth or prior depth maps.
                Shape: (B, N, H, W).
                Invalid values (e.g., sky or missing data) should be set to 0.
            mask_add_depth (torch.Tensor, optional): Mask for depth condition.
                Shape: (B, N, N).
            mask_add_ray (torch.Tensor, optional): Mask for ray/intrinsic condition.
                Shape: (B, N, N).
            mask_add_pose (torch.Tensor, optional): Mask for pose condition.
                Shape: (B, N, N).
                Note: Requires at least two frames to be True to establish a meaningful
                coordinate system (absolute pose for a single frame provides no relative constraint).

        Returns:
            dict: Model outputs containing 'points', 'conf', etc.
        """
        imgs = (imgs - self.image_mean) / self.image_std

        B, N, _, H, W = imgs.shape
        patch_h, patch_w = H // 14, W // 14

        # encode
        hidden, poses_, use_depth_mask, use_pose_mask, norm_factor = self.encode(
            imgs, 
            with_prior=with_prior, 
            depths=depths, 
            intrinsics=intrinsics, 
            poses=poses, 
            rays=rays,
            mask_add_depth=mask_add_depth,
            mask_add_ray=mask_add_ray,
            mask_add_pose=mask_add_pose,
        )
        hidden = hidden.reshape(B, N, -1, self.dec_embed_dim)

        # decode
        hidden, pos = self.decode(hidden, N, H, W, poses_, use_pose_mask)

        # # head
        outputs = self.forward_head(hidden, pos, B, N, H, W, patch_h, patch_w)

        return outputs
    
    def encode(
        self, 
        imgs, 
        with_prior=True,
        depths=None,
        rays=None,
        intrinsics=None,
        poses=None,
        mask_add_depth=None,
        mask_add_ray=None,
        mask_add_pose=None,
    ):
        B, N, _, H, W = imgs.shape
        device = imgs.device

        # encode by dinov2
        imgs = imgs.reshape(B*N, _, H, W)
        hidden = self.encoder(imgs, is_training=True)["x_norm_patchtokens"]

        if self.use_multimodal:
            with torch.amp.autocast(device_type='cuda', enabled=False):
                if with_prior is True:
                    p_depth = p_ray = p_pose = 1.0
                else:
                    p_depth = p_ray = p_pose = 0.0

                if depths is None:
                    p_depth = 0.0
                    depths = torch.zeros((B, N, H, W), device=imgs.device)

                if rays is not None:
                    rays = rays[..., :2] / (rays[..., 2:3] + 1e-6)
                else:
                    if intrinsics is None:
                        p_ray = 0.0
                        rays = torch.zeros((B, N, H, W, 2), device=imgs.device)
                    else:
                        pix = torch.from_numpy(get_pixel(H, W).T.reshape(H, W, 3)).to(device).float()[None].repeat(B, 1, 1, 1)
                        rays = torch.einsum('bnij, bhwj -> bnhwi', torch.inverse(intrinsics), pix)[..., :2]
                        # rays = F.normalize(rays, dim=-1).reshape(B, N, H, W, 3)                   # don't normalize, so the pred['xy'] is the same as input rays

                if poses is None:
                    p_pose = 0.0
                    poses = torch.eye(4, device=device)[None, None].repeat(B, N, 1, 1)
                else:
                    assert rays is not None                     # rays should be along with poses
                    
                if mask_add_depth is None:
                    mask_add_depth = torch.rand((B, N), device=device) <= p_depth
                if mask_add_ray is None:
                    mask_add_ray = torch.rand((B, N), device=device) <= p_ray
                if mask_add_pose is None:
                    mask_add_pose = torch.rand((B, N), device=device) <= p_pose

                # pose is injected relatively. so at least two frame should be true.
                num_valid_pose = mask_add_pose.sum(dim=1)
                bad_indices = (num_valid_pose == 1)
                mask_add_pose[bad_indices] = False

                # normalize depth and pose
                normalized_depths, dep_median = self.normalize_depth(depths, method='mean')
                scale_aug = 0.8 + torch.rand((B,), device=device) * 0.4
                normalized_depths /= scale_aug.view(B, 1, 1, 1)
                dep_median *= scale_aug

                depths_masks = (normalized_depths > 0).float()
                depths_masks = depths_masks.reshape(B*N, 1, H, W)

                poses_ = torch.einsum('bij, bnjk -> bnik', se3_inverse(poses[:, 0]), poses)
                poses_[..., :3, 3] /= dep_median.view(B, 1, 1)

                # noramlize for the batch not using depth
                use_depth_batch_mask = mask_add_depth.sum(dim=1) > 0
                if (~use_depth_batch_mask).sum() > 0 and N > 1:
                    pose_scale = poses_[..., 1:, :3, 3].norm(dim=-1)

                    static_threshold = 2e-2
                    is_static_mask = pose_scale.max(dim=1)[0] < static_threshold

                    pose_scale = pose_scale.mean(dim=1)
                    scale_aug = 0.8 + torch.rand((B,), device=device) * 0.4
                    pose_scale *= scale_aug

                    final_moving_mask = torch.logical_and(~use_depth_batch_mask, ~is_static_mask)
                    poses_[final_moving_mask, ..., :3, 3] /= (pose_scale.view(B, 1, 1)[final_moving_mask] + 1e-8)
                    normalized_depths[final_moving_mask] /= (pose_scale.view(B, 1, 1, 1)[final_moving_mask] + 1e-8)

                    dep_median[final_moving_mask] *= pose_scale[final_moving_mask]

                normalized_depths = normalized_depths.reshape(B*N, 1, H, W)

            if mask_add_depth.sum() > 0:
                depth_emb = self.depth_encoder(torch.cat([normalized_depths, depths_masks], dim=1), is_training=True)["x_norm_patchtokens"] + self.depth_emb
            else:
                depth_emb = torch.zeros_like(hidden)

            if mask_add_ray.sum() > 0:
                ray_emb = self.ray_embed(rays.reshape(B*N, H, W, 2).permute(0, 3, 1, 2))
            else:
                ray_emb = torch.zeros_like(hidden)

            use_depth_mask = mask_add_depth
            use_pose_mask = mask_add_pose

            hidden = hidden + ray_emb * mask_add_ray.reshape(B*N, 1, 1)
            hidden = hidden + depth_emb * mask_add_depth.reshape(B*N, 1, 1)

            return hidden, poses_, use_depth_mask, use_pose_mask, dep_median
        
        return hidden, None, None, None, None
    
    def forward_head(self, hidden, pos, B, N, H, W, patch_h, patch_w):
        device = hidden.device
        hw = patch_h*patch_w+self.patch_start_idx

        # decode point
        ret_point = self.point_decoder(hidden, xpos=pos)

        # decode camera
        ret_camera = self.camera_decoder(hidden, xpos=pos)

        # decode metric
        pos_hw = pos.reshape(B, N*hw, -1)
        ret_metric = self.metric_decoder(self.metric_token.repeat(B, 1, 1), hidden.reshape(B, N*hw, -1), xpos=pos_hw[:, 0:1], ypos=pos_hw)

        # decode conf
        ret_conf = self.conf_decoder(hidden, xpos=pos)

        with torch.amp.autocast(device_type='cuda', enabled=False):
            xy, z = self.point_head(ret_point[:, self.patch_start_idx:].float(), patch_h=patch_h, patch_w=patch_w)
            xy = xy.permute(0, 2, 3, 1).reshape(B, N, H, W, -1)
            z = z.permute(0, 2, 3, 1).reshape(B, N, H, W, -1)

            z = torch.exp(z.clamp(max=15.0))
            local_points = torch.cat([xy * z, z], dim=-1)
            rays = F.normalize(torch.cat([xy, torch.ones_like(z)], dim=-1), dim=-1)

            camera_poses = self.camera_head(ret_camera[:, self.patch_start_idx:].float(), patch_h, patch_w).reshape(B, N, 4, 4)

            metric = self.metric_head(ret_metric.float()).reshape(B).exp()

            # conf
            conf = self.conf_head(ret_conf[:, self.patch_start_idx:].float(), patch_h=patch_h, patch_w=patch_w)[0]
            conf = conf.permute(0, 2, 3, 1).reshape(B, N, H, W, -1)

            # points
            points = torch.einsum('bnij, bnhwj -> bnhwi', camera_poses, homogenize_points(local_points))[..., :3] * metric.view(B, 1, 1, 1, 1)

            # convert camera poses to metric
            camera_poses[..., :3, 3] = camera_poses[..., :3, 3] * metric.view(B, 1, 1)

            # convert local_points to metric
            local_points = local_points * metric.view(B, 1, 1, 1, 1)

        return dict(
            points=points,
            local_points=local_points,
            rays=rays,
            conf=conf,
            camera_poses=camera_poses,  
            metric=metric,
        )


    def decode(self, hidden, N, H, W, poses, use_pose_mask):
        device = hidden.device

        if len(hidden.shape) == 4:
            B, N, hw, _ = hidden.shape
        else:
            BN, hw, _ = hidden.shape
            B = BN // N

        hidden = hidden.reshape(B*N, hw, -1)

        register_token = self.register_token.repeat(B, N, 1, 1).reshape(B*N, *self.register_token.shape[-2:])
        hidden = torch.cat([register_token, hidden], dim=1)
        hw = hidden.shape[1]
        pose_inject_blk_idx = 0

        pos = self.position_getter(B*N, H//self.patch_size, W//self.patch_size, hidden.device)
        if self.patch_start_idx > 0:
            # do not use position embedding for special tokens (camera and register tokens)
            # so set pos to 0 for the special tokens
            pos_patch = pos + 1
            pos_special = torch.zeros(B * N, self.patch_start_idx, 2).to(hidden.device).to(pos.dtype)
            pos = torch.cat([pos_special, pos_patch], dim=1)

        if self.use_multimodal:
            if use_pose_mask.sum() == B * N:
                pose_inject_mask = None
            else:
                view_interaction_mask = use_pose_mask.unsqueeze(2) & use_pose_mask.unsqueeze(1)
                token_interaction_mask = view_interaction_mask.repeat_interleave(hw - self.patch_start_idx, dim=1)
                token_interaction_mask = token_interaction_mask.repeat_interleave(hw - self.patch_start_idx, dim=2)
                pose_inject_mask = token_interaction_mask[:, None]

        for i in range(len(self.decoder)):
            blk = self.decoder[i]

            if i % 2 == 0:
                pos = pos.reshape(B*N, hw, -1)
                hidden = hidden.reshape(B*N, hw, -1)
            else:
                pos = pos.reshape(B, N*hw, -1)
                hidden = hidden.reshape(B, N*hw, -1)

            hidden = blk(hidden, xpos=pos)

            if self.use_multimodal:
                if i in [1, 9, 17, 25, 33] and use_pose_mask.sum() > 0:
                    hidden = hidden.reshape(B, N, -1, 1024)
                    poses_feat = self.pose_inject_blk[pose_inject_blk_idx](hidden[..., self.patch_start_idx:, :].reshape(B, N*(hw-self.patch_start_idx), -1), poses, H, W, H//14, W//14, attn_mask=pose_inject_mask).reshape(B, N, -1, 1024)
                    hidden[..., self.patch_start_idx:, :] += poses_feat * use_pose_mask.view(B, N, 1, 1)
                    
                    hidden = hidden.reshape(B, N*hw, -1)
                    pose_inject_blk_idx += 1

            if i == len(self.decoder) - 2:
                temp_features = hidden.clone().reshape(B*N, hw, -1)

        concatenated = torch.cat((temp_features, hidden.reshape(B*N, hw, -1)), dim=-1)

        return concatenated, pos.reshape(B*N, hw, -1)
    
    
    def normalize_depth(self, depths: torch.Tensor, method: str = 'median') -> tuple[torch.Tensor, torch.Tensor]:
        """
        Normalizes a batch of depth maps using either median or mean normalization.

        Args:
            depths (torch.Tensor): A batch of depth maps with shape [B, N, H, W].
                                Non-positive values are treated as invalid depth data.
            method (str, optional): The normalization method to use.
                                    Can be 'median' or 'mean'. Defaults to 'median'.

        Returns:
            tuple[torch.Tensor, torch.Tensor]: A tuple containing:
                - The normalized depth maps.
                - The normalization factors (medians or means) used for each batch element.

        Raises:
            ValueError: If the method is not 'median' or 'mean'.
        """
        # 确保输入是 torch.Tensor
        if not isinstance(depths, torch.Tensor):
            depths = torch.tensor(depths, dtype=torch.float32)

        if method not in ['median', 'mean']:
            raise ValueError(f"Invalid normalization method: '{method}'. Choose 'median' or 'mean'.")

        B, N, H, W = depths.shape
        epsilon = 1e-8

        # Create a mask for valid depth values (positive values)
        valid_depths = torch.where(depths > 0, depths, float('nan'))
        valid_depths_reshaped = valid_depths.view(B, -1)

        if method == 'median':
            # Calculate the median for each depth map in the batch
            factors, _ = torch.nanmedian(valid_depths_reshaped, dim=1)
        elif method == 'mean':
            # Calculate the mean for each depth map in the batch
            factors = torch.nanmean(valid_depths_reshaped, dim=1)
        
        # Handle cases where all values might be NaN (e.g., all depths are 0 or negative)
        # In such cases, use 1.0 as the normalization factor to prevent division by zero.
        factors = torch.nan_to_num(factors, nan=1.0)
        
        # Reshape factors for broadcasting during division
        factors_for_division = factors.view(B, 1, 1, 1)

        # Perform normalization, adding a small epsilon to prevent division by zero
        normalized_depths = depths / (factors_for_division + epsilon)

        return normalized_depths, factors.reshape(-1)