Skip to content

SolarRenderer: Few-Step Generative Reference Rendering

Python
5.3 KB
Raw Blame
#!/usr/bin/env python
"""Tables from eval/metrics.py results: the mean of every reading per method, and the automatic score.

    python eval/summary.py --scores ./scores --methods mine,other [--split hard|heldout] [--tag "unseen condition"]

Readings (lower is better for flicker and motion error), in four groups:
  quality      DOVER overall, LAION aesthetic, VideoScore2 visual quality (expected)
  stability    low-pass warp residual, DINOv2 adjacent-frame cosine
  structure    CoTracker3 motion error, edge F1
  first frame  DINOv2 cosine to the first frame over the last second
Held-out split: also DISTS and LPIPS against the real video (eval/gt_metrics.py), when present.

Automatic score (0-100), the score of the paper: each reading is mapped linearly from a fixed range - DOVER, edge F1,
DINOv2 adjacent-frame and first-frame similarity from [0, 1], the aesthetic score from [1, 10], VideoScore2 from
[1, 5], flicker from [0, 10] and motion error from [0, 0.5], both reversed and clipped at 0 - the readings of each
group are averaged, and the score is the mean of the four groups. It uses each method's means over the cases, so it
does not depend on the other methods. Methods are compared on the cases every listed method has.
"""
from __future__ import annotations

import argparse
import json
from pathlib import Path

READ = {  # name: (metric directory, getter, group)
    "dover_overall": ("dover", lambda j: j["dover"]["overall"], "quality"),
    "laion_aes": ("laion_aes", lambda j: j["laion_aes"], "quality"),
    "vs2_visual": ("videoscore2", lambda j: j.get("visual_quality_expected"), "quality"),
    "warp_l1_lp": ("flicker", lambda j: j["warp_l1_lp"], "stability"),
    "dino_adjacent": ("dino", lambda j: j["adjacent_cos"], "stability"),
    "motion_err": ("motion", lambda j: j["err_mean"], "structure"),
    "edge_f1": ("edges", lambda j: j["F1"], "structure"),
    "first_last1s": ("firstframe", lambda j: j["last1s_cos"], "first_frame"),
    "dists_real": ("gt", lambda j: j["dists_real"], "ground_truth"),
    "lpips_real": ("gt", lambda j: j["lpips_real"], "ground_truth"),
}
SCALE = {  # reading -> 0-100
    "dover_overall": lambda x: 100 * x,
    "laion_aes": lambda x: 100 * (x - 1) / 9,
    "vs2_visual": lambda x: 100 * (x - 1) / 4,
    "warp_l1_lp": lambda x: 100 * max(0.0, 1 - x / 10),
    "dino_adjacent": lambda x: 100 * x,
    "motion_err": lambda x: 100 * max(0.0, 1 - x / 0.5),
    "edge_f1": lambda x: 100 * x,
    "first_last1s": lambda x: 100 * x,
}
GROUPS = ("quality", "stability", "structure", "first_frame")


def score(means: dict[str, float]) -> tuple[float, ...]:
    """(score, quality, stability, structure, first frame) from a method's mean readings."""
    groups = [[SCALE[n](means[n]) for n in SCALE if READ[n][2] == g] for g in GROUPS]
    groups = [sum(v) / len(v) for v in groups]
    return (sum(groups) / len(groups), *groups)


def load(scores: Path, method: str, split: str, tag: str | None = None) -> dict[str, dict[str, float]]:
    """case id -> reading -> value"""
    out: dict[str, dict[str, float]] = {}
    for name, (metric, get, _) in READ.items():
        root = scores / metric / method / split
        for p in root.rglob("*.json") if root.exists() else []:
            try:
                j = json.loads(p.read_text())
                v = get(j)
            except (KeyError, TypeError, json.JSONDecodeError):
                continue
            if tag and metric != "gt" and tag not in (j.get("tags") or []):
                continue
            if v is not None:
                out.setdefault(str(p.relative_to(root))[:-5], {})[name] = float(v)
    if tag:  # the gt readings carry no tags; keep the cases the tagged readings selected
        out = {k: v for k, v in out.items() if set(v) - {"dists_real", "lpips_real"}}
    return out


def main():
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--scores", required=True, type=Path)
    ap.add_argument("--methods", required=True, help="comma list")
    ap.add_argument("--split", default="hard", choices=("hard", "heldout"))
    ap.add_argument("--tag", help='hard split only: the cases with this tag, e.g. "unseen condition"')
    a = ap.parse_args()
    methods = [m for m in a.methods.split(",") if m]
    data = {m: load(a.scores, m, a.split, a.tag) for m in methods}
    need = [n for n, (_, _, g) in READ.items() if g != "ground_truth" or a.split == "heldout"]
    common = set.intersection(*({k for k, v in d.items() if all(n in v for n in SCALE)} for d in data.values()))
    if not common:
        raise SystemExit("no case scored for every method")
    print(f"{a.split}{', ' + a.tag if a.tag else ''}: {len(common)} cases every method has\n")
    print(f"{'method':24s} {'score':>6s} {'qual':>6s} {'stab':>6s} {'struct':>6s} {'first':>6s}  "
          + " ".join(f"{n:>13s}" for n in need))
    rows = []
    for m, d in data.items():
        means = {}
        for n in need:
            v = [d[k][n] for k in common if n in d[k]]
            if v:
                means[n] = sum(v) / len(v)
        rows.append((score(means), m, means))
    for sc, m, means in sorted(rows, key=lambda r: -r[0][0]):
        print(f"{m:24s} " + " ".join(f"{x:6.1f}" for x in sc) + "  "
              + " ".join(f"{means[n]:13.4f}" if n in means else f"{'-':>13s}" for n in need))


if __name__ == "__main__":
    main()