Skip to content

SolarRenderer: Few-Step Generative Reference Rendering

Python
7.7 KB
Raw Blame
#!/usr/bin/env python
"""Pixel metrics against the ground truth, for the held-out split.

    python eval/gt_metrics.py --bench ./SolarRenderer-Data --renders ./renders/mymethod --method mymethod --out ./scores

Writes <out>/gt/<method>/heldout/<clip_id>.json. Render and ground truth are decoded at the ground truth's size
and cut to the shorter. Readings (lower is closer for DISTS / LPIPS, higher for PSNR / SSIM):
  dists_real / lpips_real   DISTS and LPIPS (AlexNet), 4 fps, 672 px wide, mean over frames
  dists_hr / lpips_hr       the same on the four full-size quarters of every frame (2x2 tiles of 672x384)
  psnr                      per-frame 10 log10(255^2 / MSE) on full-size RGB, all frames, averaged
  ssim                      per-pixel greyscale SSIM (11x11 Gaussian, sigma 1.5), all frames, averaged
  sharp_log / sharp_log_hr  log of (mean Laplacian variance of the render / of the ground truth), grey, 8 fps,
                            672 px and full size: 0 is the ground truth's sharpness
Each reading is also given without frame 0 (suffix _nof0): frame 0 is the given first frame, so it only measures how
faithfully that frame is copied.
"""
from __future__ import annotations

import argparse
import json
import math
import subprocess
import sys
import traceback
from pathlib import Path

import numpy as np
import torch

HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
import bench  # noqa: E402

W = 672


def size_of(path):
    probe = subprocess.run(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries", "stream=width,height",
                            "-of", "csv=p=0", path], capture_output=True, text=True).stdout.strip().split(",")
    return int(probe[0]), int(probe[1])


def decode(path, fps, gray, size):
    """Frames at fps over the whole clip at size (w, h): uint8 [T,H,W] (grey) or [T,H,W,3]."""
    w, h = size
    chain = f"fps={fps},scale={w}:{h}:flags=area,format={'gray' if gray else 'rgb24'}"
    raw = subprocess.run(["ffmpeg", "-nostdin", "-loglevel", "error", "-i", path, "-vf", chain, "-f", "rawvideo", "-"],
                         capture_output=True, check=True).stdout
    c = 1 if gray else 3
    a = np.frombuffer(raw, np.uint8)
    n = len(a) // (w * h * c)
    return a[: n * w * h * c].reshape(n, h, w) if gray else a[: n * w * h * c].reshape(n, h, w, 3)


def sizes(real):
    w, h = size_of(real)
    return (W, int(round(h * W / w / 2) * 2)), (w, h)


def sharpness(gray):
    f = gray.astype(np.float32)
    lap = -4 * f[:, 1:-1, 1:-1] + f[:, :-2, 1:-1] + f[:, 2:, 1:-1] + f[:, 1:-1, :-2] + f[:, 1:-1, 2:]
    return float(lap.var(axis=(1, 2)).mean())


def to_tensor(rgb, device):
    return torch.from_numpy(rgb).to(device).permute(0, 3, 1, 2).float().div(255.0)


@torch.no_grad()
def distances(a, b, metrics, device, tiles=False):
    n = min(len(a), len(b))
    a, b = a[:n], b[:n]
    if tiles:  # the four quarters of every frame, each as its own picture
        h, w = a.shape[1] // 2, a.shape[2] // 2
        cut = lambda v: np.concatenate([v[:, i * h:(i + 1) * h, j * w:(j + 1) * w] for i in range(2) for j in range(2)])  # noqa: E731
        a, b = cut(a), cut(b)
    out = {}
    for name, m in metrics.items():
        vals = []
        for i in range(0, len(a), 8):
            x, y = to_tensor(np.ascontiguousarray(a[i:i + 8]), device), to_tensor(np.ascontiguousarray(b[i:i + 8]), device)
            vals.append(m(x, y).flatten().float().cpu())
        out[name] = float(torch.cat(vals).mean())
    return out


def real_readings(real):
    lo, hi = sizes(real)
    return {"lo": lo, "hi": hi, "sharp": sharpness(decode(real, 8, True, lo)),
            "sharp_hr": sharpness(decode(real, 8, True, hi)),
            "rgb": decode(real, 4, False, lo), "rgb_hr": decode(real, 4, False, hi)}


def _gaussian(window: int = 11, sigma: float = 1.5) -> torch.Tensor:
    axis = torch.arange(window, dtype=torch.float32) - (window - 1) / 2
    kernel = torch.exp(-(axis**2) / (2 * sigma**2))
    kernel = kernel / kernel.sum()
    return (kernel[:, None] @ kernel[None, :])[None, None]


def ssim(made: torch.Tensor, want: torch.Tensor) -> torch.Tensor:
    window = _gaussian().to(made.device, made.dtype)
    pad = window.shape[-1] // 2
    blur = lambda x: torch.nn.functional.conv2d(x, window, padding=pad)  # noqa: E731
    mu_a, mu_b = blur(made), blur(want)
    var_a = blur(made * made) - mu_a * mu_a
    var_b = blur(want * want) - mu_b * mu_b
    covariance = blur(made * want) - mu_a * mu_b
    c1, c2 = (0.01 * 255) ** 2, (0.03 * 255) ** 2
    return ((2 * mu_a * mu_b + c1) * (2 * covariance + c2)) / ((mu_a**2 + mu_b**2 + c1) * (var_a + var_b + c2))


def psnr(a, b):
    mse = float(((a - b) ** 2).mean())
    return 10 * np.log10(255.0 ** 2 / max(mse, 1e-6))


def pixel(made, real, device):
    hi = size_of(real)
    a, b = decode(made, 24, False, hi), decode(real, 24, False, hi)
    n = min(len(a), len(b))
    ps, ss = [], []
    w = torch.tensor([0.299, 0.587, 0.114], device=device)
    for k in range(n):
        ps.append(float(psnr(a[k].astype(np.float32), b[k].astype(np.float32))))
        x = torch.from_numpy(a[k]).to(device).float() @ w
        y = torch.from_numpy(b[k]).to(device).float() @ w
        ss.append(float(ssim(x[None, None], y[None, None]).mean()))
    return {"psnr": float(np.mean(ps)), "psnr_nof0": float(np.mean(ps[1:])), "psnr_frame0": ps[0],
            "ssim": float(np.mean(ss)), "ssim_nof0": float(np.mean(ss[1:])), "ssim_frame0": ss[0], "frames": n}


def quality(made, R, metrics, device):
    lo, hi = decode(made, 4, False, R["lo"]), decode(made, 4, False, R["hi"])
    d = distances(lo, R["rgb"], metrics, device)
    dh = distances(hi, R["rgb_hr"], metrics, device, tiles=True)
    d0 = distances(lo[1:], R["rgb"][1:], metrics, device)
    dh0 = distances(hi[1:], R["rgb_hr"][1:], metrics, device, tiles=True)
    return {"dists_real": d["dists"], "dists_hr": dh["dists"], "dists_real_nof0": d0["dists"],
            "dists_hr_nof0": dh0["dists"], "lpips_real": d["lpips"], "lpips_hr": dh["lpips"],
            "lpips_real_nof0": d0["lpips"], "lpips_hr_nof0": dh0["lpips"],
            "sharp_log": math.log(sharpness(decode(made, 8, True, R["lo"])) / R["sharp"]),
            "sharp_log_hr": math.log(sharpness(decode(made, 8, True, R["hi"])) / R["sharp_hr"])}


def main():
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--bench", required=True)
    ap.add_argument("--renders", required=True)
    ap.add_argument("--method", required=True)
    ap.add_argument("--out", required=True)
    ap.add_argument("--shard", type=int, default=0)
    ap.add_argument("--shards", type=int, default=1)
    ap.add_argument("--device", default="cuda:0")
    a = ap.parse_args()
    import pyiqa

    device = torch.device(a.device)
    metrics = {"dists": pyiqa.create_metric("dists", device=device, as_loss=False),
               "lpips": pyiqa.create_metric("lpips", device=device, as_loss=False)}
    for c in bench.load(a.bench, "heldout")[a.shard::a.shards]:
        out = Path(a.out) / "gt" / a.method / "heldout" / f"{c.key}.json"
        made = c.render(a.renders)
        if out.exists() or not made.exists():
            continue
        try:
            res = {**quality(str(made), real_readings(c.ground_truth), metrics, device),
                   **pixel(str(made), c.ground_truth, device)}
        except Exception:
            print(f"SKIP {c.key}\n{traceback.format_exc()[-600:]}", flush=True)
            continue
        out.parent.mkdir(parents=True, exist_ok=True)
        out.write_text(json.dumps({"id": c.id, "method": a.method, **res}))
        print(f"{c.key} dists_hr={res['dists_hr']:.4f} psnr={res['psnr']:.2f}", flush=True)


if __name__ == "__main__":
    main()