Skip to content

armgit / 67.js

Public Imported from GitHub
0

3D engine that AI agents drive

13.0 KB
Raw Blame
# Scores each 67js render against Cycles and EEVEE with FLIP, and saves side by side images.
#   uv run --no-project --python 3.12 --with flip-evaluator --with pillow --with numpy python -I bench/reference/score.py <work directory> <output directory>
# A scene that declares brightRegion (scenes.js) also gets region-* scores over the bright pixels of its Cycles image.
# FLIP is 0 for identical images, and higher means a larger visible difference. EEVEE against Cycles is a yardstick:
# it shows what a good rasterizer scores on the same scene.
# Quantisation (#258): 67js is read back from an 8-bit PNG, so the EEVEE array is rounded to 8 bits in display space the same
# way before FLIP. Cycles stays float because it is the reference both sides are measured against. 67js is not read back as
# float because the harness reads canvas pixels, which are 8-bit by construction.
#   uv run --no-project --python 3.12 --with flip-evaluator --with pillow --with numpy python -I bench/reference/score.py --self-test
import json
import sys
import tempfile
from pathlib import Path

import flip_evaluator
import numpy as np
from PIL import Image, ImageDraw

GAMMA = 2.2
LABEL_HEIGHT = 22
PANEL_WIDTH = 480
PANEL_HEIGHT = 270


def tone_map_like_67js(linear):
    """The same curve and gamma as the post-processing shader in src/post.js, so both sides meet in one display space."""
    color = np.maximum(linear, 0)
    mapped = np.clip((color * (2.51 * color + 0.03)) / (color * (2.43 * color + 0.59) + 0.14), 0, 1)
    return mapped ** (1 / GAMMA)


def quantise_to_8_bit(display):
    """Rounds display-space values to the nearest of 256 steps, as writing an 8-bit PNG does."""
    return np.round(np.clip(display, 0, 1) * 255).astype(np.float32) / 255


def load_eevee(path):
    """An EEVEE render in the same 8-bit display space as a 67js PNG, so FLIP sees equal rounding on both sides."""
    return quantise_to_8_bit(tone_map_like_67js(np.load(path)))


def self_test():
    rng = np.random.default_rng(258)
    linear = rng.random((16, 16, 3), dtype=np.float32) * 3
    with tempfile.TemporaryDirectory() as directory:
        path = Path(directory) / "scene-eevee.npy"
        np.save(path, linear)
        loaded = load_eevee(path)
        png_path = Path(directory) / "scene.png"
        Image.fromarray(np.round(np.clip(tone_map_like_67js(linear), 0, 1) * 255).astype(np.uint8)).save(png_path)
        written = load_png(png_path)
    assert np.array_equal(loaded, written), "the EEVEE array does not match what an 8-bit PNG of it reads back as"
    assert np.array_equal(np.round(loaded * 255), np.round(loaded * 255).astype(np.uint8)), "values are not whole 8-bit steps"
    assert np.array_equal(quantise_to_8_bit(loaded), loaded), "quantising twice changed the values"
    float_display = tone_map_like_67js(linear)
    assert 0 < np.abs(loaded - float_display).max() <= 0.5 / 255 + 1e-6, "the rounding error is outside half a step"
    print("score self-test ok")


def load_png(path):
    return np.asarray(Image.open(path).convert("RGB"), dtype=np.float32) / 255


COVERAGE_THRESHOLD = 0.03


def asset_coverage_mask(background, asset_render):
    """The pixels the asset changes, so a score is not diluted by the sky that both sides draw the same way."""
    return np.abs(asset_render - background).max(axis=2) > COVERAGE_THRESHOLD


def bright_region_mask(cycles, region):
    """The pixels where the Cycles image is bright, grown by a few pixels, so a mostly black frame can not hide wrong shafts.
    The Cycles image picks the region, so no renderer under test decides what is scored."""
    luminance = cycles @ np.array([0.2126, 0.7152, 0.0722], dtype=np.float32)
    bright = luminance > region["luminance"]
    radius = int(region["dilation"])
    height, width = bright.shape
    padded = np.pad(bright, radius)
    grown = np.zeros_like(bright)
    for shift_y in range(-radius, radius + 1):
        for shift_x in range(-radius, radius + 1):
            if shift_y * shift_y + shift_x * shift_x <= radius * radius:
                grown |= padded[radius + shift_y:radius + shift_y + height, radius + shift_x:radius + shift_x + width]
    return grown


def masked_flip_score(reference, test, mask):
    error_map, _, _ = flip_evaluator.evaluate(reference.astype(np.float32), test.astype(np.float32), "LDR", inputsRGB=True)
    return round(float(error_map[mask].mean()), 4)


def flip_score(reference, test):
    _, mean_error, _ = flip_evaluator.evaluate(reference.astype(np.float32), test.astype(np.float32), "LDR", inputsRGB=True)
    return round(float(mean_error), 4)


def panel(pixels, label):
    image = Image.fromarray((np.clip(pixels, 0, 1) * 255).astype(np.uint8)).resize((PANEL_WIDTH, PANEL_HEIGHT), Image.LANCZOS)
    framed = Image.new("RGB", (PANEL_WIDTH, PANEL_HEIGHT + LABEL_HEIGHT), (20, 20, 20))
    framed.paste(image, (0, LABEL_HEIGHT))
    ImageDraw.Draw(framed).text((6, 5), label, fill=(240, 240, 240))
    return framed


def side_by_side(cycles, eevee, gpu, scores):
    error_map, _, _ = flip_evaluator.evaluate(cycles.astype(np.float32), gpu.astype(np.float32), "LDR", inputsRGB=True)
    panels = [
        panel(cycles, "Cycles"),
        panel(eevee, f"EEVEE (FLIP {scores['eevee-vs-cycles']})"),
        panel(gpu, f"67js GPU (FLIP {scores['gpu-vs-cycles']})"),
        panel(error_map, "FLIP error, 67js GPU against Cycles"),
    ]
    sheet = Image.new("RGB", (PANEL_WIDTH * 2, (PANEL_HEIGHT + LABEL_HEIGHT) * 2))
    for index, item in enumerate(panels):
        sheet.paste(item, ((index % 2) * PANEL_WIDTH, (index // 2) * (PANEL_HEIGHT + LABEL_HEIGHT)))
    return sheet


def score_path_tracer(work_directory, output_directory, backend_suffix=""):
    """FLIP of each path traced render against the Cycles render of the same scene, with the time it took."""
    timings = json.loads((work_directory / f"pathtracer{backend_suffix}-timings.json").read_text(encoding="utf-8"))
    results = {}
    for scene_id in sorted(timings):
        cycles = tone_map_like_67js(np.load(work_directory / f"{scene_id}-cycles.npy"))
        traced = load_png(work_directory / f"{scene_id}-pathtracer{backend_suffix}.png")
        results[scene_id] = {"pathtracer-vs-cycles": flip_score(cycles, traced), **timings[scene_id]}
    report = {"metric": "FLIP (LDR), mean error against Cycles, lower is better", "scores": results}
    (output_directory / f"path-tracer{backend_suffix}.json").write_text(json.dumps(report, indent=1) + "\n", encoding="utf-8")
    print("scene".ljust(20) + "pathtracer-vs-cycles".rjust(22) + "seconds".rjust(10))
    for scene_id, entry in results.items():
        print(scene_id.ljust(20) + f"{entry['pathtracer-vs-cycles']:.4f}".rjust(22) + f"{entry['seconds']:.1f}".rjust(10))


def score_baked(work_directory, output_directory, scene_ids):
    """FLIP of the GPU render of each baked scene against Cycles and EEVEE, into bake.json. The CPU render is not needed."""
    images_directory = output_directory / "images"
    images_directory.mkdir(parents=True, exist_ok=True)
    results = {}
    for scene_id in scene_ids:
        cycles = tone_map_like_67js(np.load(work_directory / f"{scene_id}-cycles.npy"))
        eevee = load_eevee(work_directory / f"{scene_id}-eevee.npy")
        gpu = load_png(work_directory / f"{scene_id}-gpu.png")
        scores = {
            "gpu-vs-cycles": flip_score(cycles, gpu),
            "gpu-vs-eevee": flip_score(eevee, gpu),
            "eevee-vs-cycles": flip_score(cycles, eevee),
        }
        region = json.loads((work_directory / "scenes" / f"{scene_id}.json").read_text(encoding="utf-8")).get("brightRegion")
        if region:
            mask = bright_region_mask(cycles, region)
            scores["region-pixels"] = round(float(mask.mean()), 4)
            scores["region-gpu-vs-cycles"] = masked_flip_score(cycles, gpu, mask)
            scores["region-gpu-vs-eevee"] = masked_flip_score(eevee, gpu, mask)
            scores["region-eevee-vs-cycles"] = masked_flip_score(cycles, eevee, mask)
        results[scene_id] = scores
        side_by_side(cycles, eevee, gpu, scores).save(images_directory / f"{scene_id}-baked.jpg", quality=80)
    graphics = json.loads((work_directory / "graphics.json").read_text(encoding="utf-8"))
    timings = json.loads((work_directory / "blender-timings.json").read_text(encoding="utf-8"))
    report = {
        "metric": "FLIP (LDR), mean error of the baked GPU render against Cycles, lower is better",
        "blenderVersion": timings["blenderVersion"],
        "graphics": {"gpu": graphics["gpu"]},
        "scores": results,
    }
    (output_directory / "bake.json").write_text(json.dumps(report, indent=1) + "\n", encoding="utf-8")
    print("scene".ljust(20) + "".join(column.rjust(18) for column in ["gpu-vs-cycles", "gpu-vs-eevee", "eevee-vs-cycles"]))
    for scene_id, scores in results.items():
        print(scene_id.ljust(20) + "".join(f"{scores[column]:.4f}".rjust(18) for column in ["gpu-vs-cycles", "gpu-vs-eevee", "eevee-vs-cycles"]))


def main():
    if "--self-test" in sys.argv:
        self_test()
        return
    if "--bake" in sys.argv:
        score_baked(Path(sys.argv[1]), Path(sys.argv[2]), sys.argv[sys.argv.index("--bake") + 1].split(","))
        return
    if "--path-tracer" in sys.argv:
        score_path_tracer(Path(sys.argv[1]), Path(sys.argv[2]), "-gpu" if "--gpu-tracer" in sys.argv else "")
        return
    # Khronos assets and cloth sheets sit in front of a sky that both sides draw alike, so they also get a score over the
    # pixels the scene covers.
    set_name = "khronos" if "--khronos" in sys.argv else "cloth" if "--cloth" in sys.argv else "grand" if "--grand" in sys.argv else None
    khronos = set_name in ("khronos", "cloth")
    work_directory = Path(sys.argv[1])
    output_directory = Path(sys.argv[2])
    images_directory = output_directory / "images"
    images_directory.mkdir(parents=True, exist_ok=True)
    scene_ids = sorted(path.stem for path in (work_directory / "scenes").glob("*.json"))
    results = {}
    for scene_id in scene_ids:
        cycles = tone_map_like_67js(np.load(work_directory / f"{scene_id}-cycles.npy"))
        eevee = load_eevee(work_directory / f"{scene_id}-eevee.npy")
        gpu = load_png(work_directory / f"{scene_id}-gpu.png")
        cpu = load_png(work_directory / f"{scene_id}-cpu.png")
        scores = {
            "gpu-vs-cycles": flip_score(cycles, gpu),
            "gpu-vs-eevee": flip_score(eevee, gpu),
            "cpu-vs-cycles": flip_score(cycles, cpu),
            "cpu-vs-eevee": flip_score(eevee, cpu),
            "eevee-vs-cycles": flip_score(cycles, eevee),
            "gpu-vs-cpu": flip_score(cpu, gpu),
        }
        if khronos:
            mask = asset_coverage_mask(load_png(work_directory / f"{scene_id}-background.png"), gpu)
            scores["asset-pixels"] = round(float(mask.mean()), 4)
            scores["asset-gpu-vs-cycles"] = masked_flip_score(cycles, gpu, mask)
            scores["asset-gpu-vs-eevee"] = masked_flip_score(eevee, gpu, mask)
            scores["asset-eevee-vs-cycles"] = masked_flip_score(cycles, eevee, mask)
        region = json.loads((work_directory / "scenes" / f"{scene_id}.json").read_text(encoding="utf-8")).get("brightRegion")
        if region and not set_name:
            mask = bright_region_mask(cycles, region)
            scores["region-pixels"] = round(float(mask.mean()), 4)
            scores["region-gpu-vs-cycles"] = masked_flip_score(cycles, gpu, mask)
            scores["region-gpu-vs-eevee"] = masked_flip_score(eevee, gpu, mask)
            scores["region-eevee-vs-cycles"] = masked_flip_score(cycles, eevee, mask)
        results[scene_id] = scores
        image_name = f"{set_name}-{scene_id}" if set_name else scene_id
        side_by_side(cycles, eevee, gpu, scores).save(images_directory / f"{image_name}.jpg", quality=80)
    timings = json.loads((work_directory / "blender-timings.json").read_text(encoding="utf-8"))
    graphics = json.loads((work_directory / "graphics.json").read_text(encoding="utf-8"))
    report = {"metric": "FLIP (LDR), mean error, lower is better", **timings, "graphics": graphics, "scores": results}
    report_name = f"{set_name}.json" if set_name else "baseline.json"
    (output_directory / report_name).write_text(json.dumps(report, indent=1) + "\n", encoding="utf-8")
    columns = ["gpu-vs-cycles", "gpu-vs-eevee", "cpu-vs-cycles", "cpu-vs-eevee", "eevee-vs-cycles", "gpu-vs-cpu"]
    if khronos:
        columns += ["asset-pixels", "asset-gpu-vs-cycles", "asset-gpu-vs-eevee", "asset-eevee-vs-cycles"]
    print("scene".ljust(20) + "".join(column.rjust(17) for column in columns))
    for scene_id, scores in results.items():
        print(scene_id.ljust(20) + "".join(f"{scores[column]:.4f}".rjust(17) for column in columns))
    region_columns = ["region-pixels", "region-gpu-vs-cycles", "region-gpu-vs-eevee", "region-eevee-vs-cycles"]
    region_results = {scene_id: scores for scene_id, scores in results.items() if "region-pixels" in scores}
    if region_results:
        print()
        print("scene".ljust(20) + "".join(column.rjust(24) for column in region_columns))
        for scene_id, scores in region_results.items():
            print(scene_id.ljust(20) + "".join(f"{scores[column]:.4f}".rjust(24) for column in region_columns))


main()