Skip to content

SUPRA-Tex: 3D Generation with Super-Aligned Texture

9.7 KB
Raw Blame
"""CUDA implementation of the v4 occupancy search and normal rendering."""

from __future__ import annotations

import math
import time

import numpy as np

from .camera import (BORDER_PX, COARSE_RES, FINE_RES, NEAR, FAR, PITCH_RANGE_DEG,
                     RESOLUTION, SCALE_RANGE, YAW_RANGE_DEG, normal_sign_check,
                     occupancy_score)


def _eyes(distance, params, target, device):
    import torch
    params = torch.as_tensor(params, dtype=torch.float32, device=device)
    yaw, pitch = math.pi + torch.deg2rad(params[:, 0]), torch.deg2rad(params[:, 1])
    offsets = (distance * params[:, 2, None]) * torch.stack(
        [torch.sin(yaw) * torch.cos(pitch), torch.sin(pitch), torch.cos(yaw) * torch.cos(pitch)], dim=-1)
    return offsets + torch.as_tensor(target, dtype=torch.float32, device=device)


def _look_at(eyes, target):
    import torch
    import utils3d
    target = torch.as_tensor(target, dtype=eyes.dtype, device=eyes.device).expand_as(eyes)
    up = torch.tensor([0.0, 1.0, 0.0], dtype=eyes.dtype, device=eyes.device).expand_as(eyes)
    return utils3d.torch.extrinsics_look_at(eyes, target, up)


def _border(masks):
    return (masks[..., :BORDER_PX, :].flatten(-2).any(-1)
            | masks[..., -BORDER_PX:, :].flatten(-2).any(-1)
            | masks[..., :, :BORDER_PX].flatten(-2).any(-1)
            | masks[..., :, -BORDER_PX:].flatten(-2).any(-1))


class SilhouetteRasterizer:
    """Binary silhouettes using precisely the local MeshRenderer projection."""

    def __init__(self, vertices, faces, intrinsic):
        import torch
        import nvdiffrast.torch as dr
        from supratex.renderers.mesh_renderer import intrinsics_to_projection
        self.dr, self.faces = dr, faces
        self.context = dr.RasterizeCudaContext(device=vertices.device)
        self.vertices = torch.cat([vertices, torch.ones_like(vertices[:, :1])], dim=1).contiguous()
        self.projection = intrinsics_to_projection(intrinsic, NEAR, FAR)

    def masks(self, extrinsics, resolution):
        import torch
        masks = []
        # One camera per raster call bounds peak triangle-buffer memory.
        for extrinsic in extrinsics:
            full = self.projection @ extrinsic
            clip = (self.vertices @ full.T).unsqueeze(0).contiguous()
            raster, _ = self.dr.rasterize(self.context, clip, self.faces, (resolution, resolution))
            masks.append(raster[..., 3] > 0)
        return torch.cat(masks, dim=0)


class BackSearch:
    """Deterministic 256-pixel grid followed by 512-pixel pattern search."""

    def __init__(self, raster, camera, front_extrinsic, device):
        self.raster, self.camera = raster, camera
        self.front_extrinsic, self.device = front_extrinsic, device
        self.front_cache = {}

    def front(self, resolution):
        if resolution not in self.front_cache:
            mask = self.raster.masks(self.front_extrinsic[None], resolution)[0]
            self.front_cache[resolution] = (float(mask.float().mean()), bool(_border(mask)))
        return self.front_cache[resolution]

    def score(self, params, resolution):
        eyes = _eyes(self.camera["distance"], params, self.camera["back_look_at"], self.device)
        masks = self.raster.masks(_look_at(eyes, self.camera["back_look_at"]), resolution)
        occupancy = masks.flatten(1).float().mean(1).cpu().numpy()
        border = _border(masks).cpu().numpy()
        front_occupancy, front_border = self.front(resolution)
        params = np.asarray(params, dtype=np.float32)
        scores = occupancy_score(front_occupancy, occupancy, params[:, 0], params[:, 1], params[:, 2],
                                 border, front_border)
        return scores, occupancy

    def run(self):
        start = time.monotonic()
        yaws = np.arange(-YAW_RANGE_DEG, YAW_RANGE_DEG + 1e-6, 5.0)
        pitches = np.arange(-PITCH_RANGE_DEG, PITCH_RANGE_DEG + 1e-6, 5.0)
        scales = np.round(np.arange(SCALE_RANGE[0], SCALE_RANGE[1] + 1e-6, 0.05), 4)
        grid = np.array([(yaw, pitch, scale) for yaw in yaws for pitch in pitches for scale in scales],
                        dtype=np.float32)
        scores, occupancy = self.score(grid, COARSE_RES)
        # Stable ordering makes exact ties reproducible across repeated runs.
        order = np.argsort(-scores, kind="stable")
        top5 = [{"dyaw_deg": float(grid[index, 0]), "pitch_deg": float(grid[index, 1]),
                 "scale": float(grid[index, 2]), "back_occupancy": float(occupancy[index]),
                 "score": float(scores[index])} for index in order[:5]]
        best = grid[order[0]].copy()
        best_score = float(self.score(best[None], FINE_RES)[0][0])
        steps = np.array([2.5, 2.5, 0.025], dtype=np.float32)
        lower = np.array([-YAW_RANGE_DEG, -PITCH_RANGE_DEG, SCALE_RANGE[0]], dtype=np.float32)
        upper = np.array([YAW_RANGE_DEG, PITCH_RANGE_DEG, SCALE_RANGE[1]], dtype=np.float32)
        offsets = np.array([(a, b, c) for a in (-1, 0, 1) for b in (-1, 0, 1) for c in (-1, 0, 1)
                            if (a, b, c) != (0, 0, 0)], dtype=np.float32)
        evaluations = 0
        for _ in range(4):
            for _ in range(8):
                candidates = np.clip(best[None] + offsets * steps[None], lower, upper)
                scores, _ = self.score(candidates, FINE_RES)
                evaluations += len(candidates)
                winner = int(np.argmax(scores))
                if float(scores[winner]) <= best_score + 1e-6:
                    break
                best, best_score = candidates[winner].copy(), float(scores[winner])
            steps *= 0.5
        return {"dyaw_deg": float(best[0]), "pitch_deg": float(best[1]), "scale": float(best[2]),
                "fine_score": best_score,
                "coarse_grid": {"dyaw_deg": [-YAW_RANGE_DEG, YAW_RANGE_DEG, 5.0],
                                "pitch_deg": [-PITCH_RANGE_DEG, PITCH_RANGE_DEG, 5.0],
                                "scale": [SCALE_RANGE[0], SCALE_RANGE[1], 0.05],
                                "count": len(grid), "resolution": COARSE_RES},
                "coarse_top5": top5,
                "refine": {"resolution": FINE_RES, "evaluations": evaluations,
                           "final_step": (steps * 2).astype(float).tolist()},
                "seconds": round(time.monotonic() - start, 3)}


def _normal(renderer, mesh, extrinsic, intrinsic):
    import torch
    result = renderer.render(mesh, extrinsic, intrinsic, return_types=["mask", "normal"])
    cv = result["normal"] * 2.0 - 1.0
    normals = torch.nn.functional.normalize(torch.stack([-cv[0], cv[1], cv[2]], dim=0), dim=0)
    mask = result["mask"].reshape(RESOLUTION, RESOLUTION)
    rgb = (normals * 0.5 + 0.5).clamp(0, 1) * mask[None]
    rgb = (rgb.permute(1, 2, 0).cpu().numpy() * 255).round().astype(np.uint8)
    return rgb, normals.cpu().numpy(), mask.cpu().numpy()


def render_geometry(payload, camera, *, device="cuda", simplify=0):
    """Render a single mesh; image writes and provenance are owned by pack.py."""
    import torch
    import utils3d
    from supratex.renderers import MeshRenderer
    from supratex.representations import Mesh
    if torch.device(device).type != "cuda" or not torch.cuda.is_available():
        raise RuntimeError("back-view normal rendering requires CUDA and nvdiffrast")
    with torch.cuda.device(torch.device(device)), torch.inference_mode():
        vertices = payload["verts"].detach().to(device=device, dtype=torch.float32).contiguous()
        faces = payload["faces"].detach().to(device=device, dtype=torch.int32).contiguous()
        mesh = Mesh(vertices, faces)
        if simplify and len(faces) > simplify:
            mesh.simplify(target=simplify)
            vertices, faces = mesh.vertices.contiguous(), mesh.faces.int().contiguous()
        if camera["explicit"]:
            front = torch.as_tensor(camera["extrinsic"], device=device, dtype=torch.float32)
            intrinsic = torch.as_tensor(camera["intrinsic"], device=device, dtype=torch.float32)
        else:
            eye = torch.tensor(camera["front_eye"], device=device, dtype=torch.float32)
            front = _look_at(eye[None], camera["front_look_at"])[0]
            fov = torch.tensor(camera["fov"], device=device, dtype=torch.float32)
            intrinsic = utils3d.torch.intrinsics_from_fov_xy(fov, fov)
        raster = SilhouetteRasterizer(vertices, faces, intrinsic)
        search = BackSearch(raster, camera, front, device).run()
        params = [[0.0, 0.0, 1.0], [search["dyaw_deg"], search["pitch_deg"], search["scale"]]]
        eyes = _eyes(camera["distance"], params, camera["back_look_at"], device)
        back_extrinsics = _look_at(eyes, camera["back_look_at"])
        masks = raster.masks(back_extrinsics, RESOLUTION).cpu().numpy()
        front_silhouette = raster.masks(front[None], RESOLUTION)[0].cpu().numpy()
        renderer = MeshRenderer({"resolution": RESOLUTION, "near": NEAR, "far": FAR,
                                 "ssaa": 1, "antialias": True}, device=device)
        back_rgb, back_normals, back_mask = _normal(renderer, mesh, back_extrinsics[1], intrinsic)
        front_rgb, front_normals, front_mask = _normal(renderer, mesh, front, intrinsic)
        return {"front_normal": front_rgb, "back_normal": back_rgb, "back_mask": back_mask > 0.5,
                "front_silhouette": front_silhouette, "exact_back_silhouette": masks[0],
                "back_silhouette": masks[1], "search": search, "faces": len(faces), "simplify": simplify,
                "front_extrinsic": front.cpu().numpy(), "back_extrinsic": back_extrinsics[1].cpu().numpy(),
                "back_eye": eyes[1].cpu().numpy(), "intrinsic": intrinsic.cpu().numpy(),
                "normal_sign_check": {"front": normal_sign_check(front_normals, front_mask),
                                      "back": normal_sign_check(back_normals, back_mask)}}