Skip to content

3D engine that AI agents drive

5.3 KB
Raw Blame
"""FLIP error of one scene split by what each pixel shows (a part, the open floor, the sky), for Crucible, EEVEE and Cycles.

Usage: uv run --no-project --python 3.12 --with flip-evaluator --with pillow --with numpy python -I bench/reference/region-errors.py <scene> [gpu|cpu]

Reads the work directory of run.mjs (artifacts/reference): the scene description, the Cycles and EEVEE arrays and the Crucible render.
Each part's triangles are drawn back to front into a label image with the scene's camera, so a pixel belongs to the nearest part.
The floor is drawn only as the pixels below the horizon that no other part covers, because its quad reaches behind the camera.
`share` is the part of the frame a region covers, `error` its mean FLIP, and `contribution` how much of the frame's mean FLIP it holds."""
import base64
import json
import sys
from pathlib import Path

import flip_evaluator
import numpy as np
from PIL import Image, ImageDraw

GAMMA = 2.2  # as in score.py, which runs its scoring on import and so cannot be imported
WORK_DIRECTORY = Path("artifacts/reference")
WIDTH, HEIGHT = 960, 540
OPEN_FLOOR_LABEL = 99
LUMINANCE_WEIGHTS = np.array([0.2126, 0.7152, 0.0722], dtype=np.float32)
MINIMUM_DEPTH = 0.1


def tone_map_like_crucible(linear):
    color = np.maximum(linear, 0)
    mapped = np.clip((color * (2.51 * color + 0.03)) / (color * (2.43 * color + 0.59) + 0.14), 0, 1)
    return mapped ** (1 / GAMMA)


def load_png(path):
    return np.asarray(Image.open(path).convert("RGB"), dtype=np.float32) / 255


def decode(entry, dtype):
    return np.frombuffer(base64.b64decode(entry["base64"]), dtype=dtype)


def camera_projection(camera):
    eye = np.array(camera["eye"])
    forward = np.array(camera["target"]) - eye
    forward /= np.linalg.norm(forward)
    right = np.cross(forward, np.array(camera["up"]))
    right /= np.linalg.norm(right)
    up = np.cross(right, forward)
    tangent = np.tan(np.radians(camera["fov"]) / 2)

    def project(points):
        relative = points - eye
        across, above, depth = relative @ right, relative @ up, relative @ forward
        pixel_x = (across / (depth * tangent * WIDTH / HEIGHT) * 0.5 + 0.5) * WIDTH
        pixel_y = (0.5 - above / (depth * tangent) * 0.5) * HEIGHT
        return pixel_x, pixel_y, depth

    return project


def label_image(description):
    project = camera_projection(description["camera"])
    triangles = []
    for part_index, part in enumerate(description["parts"]):
        geometry = description["geometries"][part["geometry"]]
        positions = decode(geometry["positions"], np.float32).reshape(-1, 3)
        indices = decode(geometry["indices"], np.uint32).reshape(-1, 3)
        matrix = np.array(part["matrix"]).reshape(4, 4)
        world = positions @ matrix[:3, :3] + matrix[3, :3]
        pixel_x, pixel_y, depth = project(world)
        for triangle in indices:
            if (depth[triangle] < MINIMUM_DEPTH).any():
                continue
            corners = [(pixel_x[corner], pixel_y[corner]) for corner in triangle]
            triangles.append((depth[triangle].mean(), part_index + 1, corners))
    triangles.sort(key=lambda entry: -entry[0])
    image = Image.new("I", (WIDTH, HEIGHT), 0)
    drawing = ImageDraw.Draw(image)
    for _, label, corners in triangles:
        drawing.polygon(corners, fill=label)
    labels = np.array(image)
    horizon_y = project(np.array([[0.0, 0.0, -100000.0]]))[1][0]
    rows = np.arange(HEIGHT)[:, None] * np.ones((1, WIDTH))
    return np.where((labels == 0) & (rows > horizon_y), OPEN_FLOOR_LABEL, labels)


def main():
    scene = sys.argv[1]
    backend = sys.argv[2] if len(sys.argv) > 2 else "gpu"
    description = json.loads((WORK_DIRECTORY / "scenes" / f"{scene}.json").read_text(encoding="utf-8"))
    names = ["sky"] + [part["name"] for part in description["parts"]]
    labels = label_image(description)
    cycles = tone_map_like_crucible(np.load(WORK_DIRECTORY / f"{scene}-cycles.npy")).astype(np.float32)
    eevee = tone_map_like_crucible(np.load(WORK_DIRECTORY / f"{scene}-eevee.npy")).astype(np.float32)
    render = load_png(WORK_DIRECTORY / f"{scene}-{backend}.png")
    render_map, render_mean, _ = flip_evaluator.evaluate(cycles, render.astype(np.float32), "LDR", inputsRGB=True)
    eevee_map, eevee_mean, _ = flip_evaluator.evaluate(cycles, eevee, "LDR", inputsRGB=True)
    render_error = render_map.mean(axis=-1) if render_map.ndim == 3 else render_map
    eevee_error = eevee_map.mean(axis=-1) if eevee_map.ndim == 3 else eevee_map
    print(f"{scene} ({backend}): Crucible {render_mean:.4f}, EEVEE {eevee_mean:.4f}, against Cycles")
    print("region        share  error crucible/EEVEE  contribution crucible/EEVEE  luminance crucible/Cycles/EEVEE")
    for label in list(range(len(names))) + [OPEN_FLOOR_LABEL]:
        mask = labels == label
        if not mask.any():
            continue
        name = "floor (open)" if label == OPEN_FLOOR_LABEL else names[label]
        luminance = [(image @ LUMINANCE_WEIGHTS)[mask].mean() for image in (render, cycles, eevee)]
        print(f"{name:12s} {mask.mean():6.3f}  {render_error[mask].mean():.4f} {eevee_error[mask].mean():.4f}"
              f"        {render_error[mask].sum() / mask.size:.4f} {eevee_error[mask].sum() / mask.size:.4f}"
              f"            {luminance[0]:.3f} {luminance[1]:.3f} {luminance[2]:.3f}")


main()