Skip to content

[SIGGRAPH Asia 2026] 4DAnyone: Create Anyone in 4D from a Casual Monocular Video

11.8 KB
Raw Blame
"""Rerun scene geometry and native-video logging; no task or cache ownership."""

from __future__ import annotations

from fractions import Fraction
from pathlib import Path

import numpy as np

from fdanyone.errors import FourDAnyoneError
from fdanyone.skeleton.keypoints import BLUE, LINKS, VISIBLE_KEYPOINT_IDS, keypoint_color

CAMERA_PLANE_DISTANCE = 0.36


def log_video(recording, entity: str, video: Path, fps, frames: int) -> None:
    import rerun as rr

    asset = rr.AssetVideo(path=video)
    timestamps = asset.read_frame_timestamps_nanos()
    if len(timestamps) != frames:
        raise FourDAnyoneError(f"Video preview has {len(timestamps)} frames, expected {frames}.")
    recording.log(entity, asset, static=True)
    recording.send_columns(
        entity,
        indexes=[rr.TimeColumn("time", duration=np.arange(frames) / float(fps))],
        columns=rr.VideoFrameReference.columns_nanos(timestamps),
    )


def log_cameras(recording, cameras, *, videos=(), fps=None, frames=121) -> None:
    """An editable empty rig, or the fixed calibrated cameras of a result."""
    import rerun as rr

    if videos:
        for camera, video in zip(cameras, videos, strict=True):
            entity = f"world/cameras/{camera['camera_id']:02d}"
            transform = np.asarray(camera["camera_to_world"])
            recording.log(entity, rr.TransformAxes3D(0, show_frame=False), static=True)
            recording.log(entity, rr.Transform3D(mat3x3=transform[:3, :3], translation=transform[:3, 3]), static=True)
            recording.log(
                entity,
                rr.Pinhole(
                    image_from_camera=np.asarray(camera["K"]),
                    resolution=[camera["image_width"], camera["image_height"]],
                    camera_xyz=rr.ViewCoordinates.RDF,
                    image_plane_distance=CAMERA_PLANE_DISTANCE,
                    color=BLUE,
                    line_width=0,
                ),
                static=True,
            )
            log_video(recording, f"{entity}/image", video, fps, frames)
    _log_camera_frames(
        recording,
        [(camera, (camera["image_width"], camera["image_height"]), bool(videos)) for camera in cameras],
    )
    recording.log(
        "world/labels",
        rr.Points3D(
            [np.asarray(camera["camera_to_world"])[:3, 3] for camera in cameras],
            colors=BLUE,
            radii=0.018,
            labels=[f"{camera['camera_id']:02d}" for camera in cameras],
            show_labels=True,
        ),
        static=True,
    )


def blueprint(context: dict, count: int):
    import rerun.blueprint as rrb

    center = np.asarray(context["center"]) + [0, context["target_height"], 0]
    scene = rrb.Spatial3DView(
        origin="world",
        name="Scene",
        contents=[
            "world/body/**",
            "world/cameras/**",
            "world/camera_frames/**",
            "world/labels/**",
            "world/stage/**",
            "world/reference/**",
        ],
        eye_controls=rrb.EyeControls3D(
            position=center + np.array([0.85, 0.55, 1.1]) * context["radius"] * (1.25 if count >= 12 else 1),
            look_target=center,
            eye_up=[0, 1, 0],
        ),
    )
    return rrb.Blueprint(
        scene,
        rrb.TimePanel(timeline="time", play_state="playing", loop_mode="all", state="hidden"),
        rrb.BlueprintPanel(state="hidden"),
        rrb.SelectionPanel(state="hidden"),
    )


def _log_camera_frames(recording, frames: list) -> None:
    """Batch static rig geometry so repaint cost does not grow by entity per camera."""
    import rerun as rr

    entity = "world/camera_frames"
    # Only display geometry lives here, so clearing it never masks video frames.
    recording.log(entity, rr.Clear(recursive=True), static=True)
    lines, fills, bezels = [], [], []
    for camera, size, has_video in frames:
        width, height = size
        outline, bezel, faces = _rounded_frame(width, height, min(size) * 0.075, min(size) * 0.02)
        transform = np.asarray(camera["camera_to_world"])
        origin = transform[:3, 3]
        projection = np.linalg.inv(camera["K"]).T @ transform[:3, :3].T * CAMERA_PLANE_DISTANCE
        plane = np.column_stack([outline + [width / 2, height / 2], np.ones(len(outline))]) @ projection + origin
        corners = plane[np.arange(4) * (len(outline) // 4) + len(outline) // 8]
        lines.extend([np.vstack([plane, plane[0]]), *[np.stack([origin, corner]) for corner in corners]])
        if has_video:
            vertices = np.column_stack([bezel + [width / 2, height / 2], np.ones(len(bezel))]) @ projection + origin
            for direction in (-1, 1):
                bezels.append((vertices + transform[:3, 2] * (direction * 0.0005), faces))
        else:
            count = len(plane)
            fills.append(
                (
                    np.vstack([plane.mean(axis=0), plane]),
                    np.column_stack([np.zeros(count), np.arange(count) + 1, (np.arange(count) + 1) % count + 1]),
                )
            )
    recording.log(
        f"{entity}/lines",
        rr.LineStrips3D(lines, colors=BLUE, radii=rr.Radius.ui_points(0.75)),
        static=True,
    )
    for name, meshes, color in [("fill", fills, [*BLUE, 65]), ("bezel", bezels, BLUE)]:
        if not meshes:
            continue
        vertices, faces, offset = [], [], 0
        for points, triangles in meshes:
            vertices.append(points)
            faces.append(triangles + offset)
            offset += len(points)
        recording.log(
            f"{entity}/{name}",
            rr.Mesh3D(
                vertex_positions=np.concatenate(vertices),
                triangle_indices=np.concatenate(faces).astype(np.uint32),
                albedo_factor=color,
            ),
            static=True,
        )


def log_stage(recording, context: dict) -> None:
    """A thin disk matching the camera ring, just below the original ground grid."""
    import rerun as rr

    count = 128
    angles = np.arange(count) * (2 * np.pi / count)
    center = np.asarray(context["center"], dtype=np.float64) - [0, 0.005, 0]
    radius = context["radius"]
    rim = center + np.column_stack([np.cos(angles), np.zeros(count), np.sin(angles)]) * radius
    indices = np.arange(count)
    next_indices = (indices + 1) % count
    recording.log(
        "world/stage/top",
        rr.Mesh3D(
            vertex_positions=np.vstack([center, rim]),
            triangle_indices=np.column_stack([np.zeros(count), next_indices + 1, indices + 1]).astype(np.uint32),
            vertex_normals=np.tile([0, 1, 0], (count + 1, 1)),
            albedo_factor=[32, 34, 39],
        ),
        static=True,
    )
    recording.log(
        "world/stage/edge",
        rr.Mesh3D(
            vertex_positions=np.vstack([rim, rim - [0, 0.005, 0]]),
            triangle_indices=np.vstack(
                [
                    np.column_stack([indices, next_indices, next_indices + count]),
                    np.column_stack([indices, next_indices + count, indices + count]),
                ]
            ),
            albedo_factor=[22, 24, 28],
        ),
        static=True,
    )
    recording.log(
        "world/stage/rim",
        rr.LineStrips3D([np.vstack([rim, rim[0]])], radii=rr.Radius.ui_points(0.6), colors=[68, 72, 80]),
        static=True,
    )


def _rounded_outline(width: float, height: float, radius: float) -> np.ndarray:
    centers = [(width / 2 - radius, height / 2 - radius), (-width / 2 + radius, height / 2 - radius)]
    centers += [(-width / 2 + radius, -height / 2 + radius), (width / 2 - radius, -height / 2 + radius)]
    angles = np.linspace(0, np.pi / 2, 13)[None, :] + np.arange(4)[:, None] * (np.pi / 2)
    arcs = np.stack([np.cos(angles), np.sin(angles)], axis=-1)
    return (np.asarray(centers)[:, None, :] + radius * arcs).reshape(-1, 2)


def _rounded_frame(width: float, height: float, radius: float, border: float):
    inner = _rounded_outline(width, height, radius)
    # A tighter outer curve keeps the video's square corners under even a thin bezel.
    outer = _rounded_outline(width + border * 2, height + border * 2, min(radius + border, border * 3))
    count = len(inner)
    indices, next_indices = np.arange(count), (np.arange(count) + 1) % count
    faces = np.vstack(
        [
            np.column_stack([indices, next_indices, next_indices + count]),
            np.column_stack([indices, next_indices + count, indices + count]),
        ]
    )
    return outer, np.vstack([outer, inner]), faces


def log_source_card(recording, context: dict, video: Path, source_info: dict) -> None:
    """A rounded video screen: full brightness toward +Z and dimmed on the back."""
    import rerun as rr

    width, height = source_info["width"], source_info["height"]
    card_height = 2 * context["target_height"]
    card_width = card_height * width / height
    radius, border = min(card_width, card_height) * np.array([0.06, 0.012])
    center = np.asarray(context["center"]) + [0, card_height / 2 + border, 0]
    outer, bezel, faces = _rounded_frame(card_width, card_height, radius, border)
    count = len(outer)
    indices, next_indices = np.arange(count), (np.arange(count) + 1) % count
    frame = np.column_stack([bezel, np.zeros(count * 2)]) + center
    # One double-sided video needs one decoder. A back-facing translucent mesh
    # dims its reverse side; the bezels mask the rectangular video corners.
    recording.log(
        "world/reference/back_tint",
        rr.Mesh3D(
            vertex_positions=np.vstack([center, np.column_stack([outer, np.zeros(count)]) + center]) - [0, 0, 0.001],
            triangle_indices=np.column_stack([np.zeros(count), indices + 1, next_indices + 1]).astype(np.uint32),
            albedo_factor=[12, 14, 18, 178],
            face_rendering=rr.components.MeshFaceRendering.Back,
        ),
        static=True,
    )
    for side, direction in [("front", 1), ("back", -1)]:
        recording.log(
            f"world/reference/{side}_bezel",
            rr.Mesh3D(
                vertex_positions=frame + [0, 0, direction * 0.002], triangle_indices=faces, albedo_factor=[30, 33, 39]
            ),
            static=True,
        )
    entity = "world/reference/screen"
    recording.log(entity, rr.TransformAxes3D(0, show_frame=False), static=True)
    recording.log(
        entity,
        rr.Transform3D(mat3x3=np.diag([1, -1, -1]), translation=center + [0, 0, card_height]),
        static=True,
    )
    recording.log(
        entity,
        rr.Pinhole(
            focal_length=height,
            resolution=[width, height],
            camera_xyz=rr.ViewCoordinates.RDF,
            image_plane_distance=card_height,
            line_width=0,
        ),
        static=True,
    )
    log_video(recording, f"{entity}/video", video, Fraction(source_info["fps"]), source_info["frames"])


def log_body(recording, body: dict, fps, check_cancelled) -> None:
    import rerun as rr

    point_ids = sorted(VISIBLE_KEYPOINT_IDS)
    recording.log(
        "world/body/mesh",
        rr.Mesh3D.from_fields(triangle_indices=body["faces"], albedo_factor=[180, 180, 180, 75]),
        static=True,
    )
    recording.log(
        "world/body/skeleton",
        rr.LineStrips3D.from_fields(radii=0.008, colors=[color for _, _, _, color, _ in LINKS]),
        static=True,
    )
    recording.log(
        "world/body/joints",
        rr.Points3D.from_fields(radii=0.012, colors=[keypoint_color(i) for i in point_ids]),
        static=True,
    )
    for index, vertices in enumerate(body["vertices"]):
        check_cancelled()
        recording.set_time("time", duration=index / float(fps))
        keypoints = body["keypoints"][index]
        recording.log("world/body/mesh", rr.Mesh3D.from_fields(vertex_positions=vertices))
        recording.log(
            "world/body/skeleton", rr.LineStrips3D.from_fields(strips=[keypoints[[a, b]] for _, a, b, _, _ in LINKS])
        )
        recording.log("world/body/joints", rr.Points3D.from_fields(positions=keypoints[point_ids]))