Skip to content

SolarRenderer: Few-Step Generative Reference Rendering

Python
23.0 KB
Raw Blame
#!/usr/bin/env python
"""Automatic metrics of the SolarRenderer benchmark, the same on both splits.

    python eval/metrics.py --bench ./SolarRenderer-Data --renders ./renders/mymethod --method mymethod \
        --out ./scores [--split hard|heldout|all] [--metrics dover,laion_aes,...] [--shard k --shards n]

For every case with a render, computes the missing metrics and writes <out>/<metric>/<method>/<case key>.json, where
the case key is hard/<case_id> or heldout/<clip_id>. Run one process per GPU with --shard / --shards.
VideoScore2 needs its own environment (transformers 4.53); run it as a separate pass with --metrics videoscore2.
Models are set up by eval/setup.sh.

Readings, grouped into the four dimensions of eval/summary.py:
  quality     dover        DOVER (ICCV 2023), technical / aesthetic / overall; the repo's preprocessing (val-l1080p
                           views), numpy seeded with 0 before every video, score-level fusion (see dover_score.py)
              laion_aes    LAION aesthetic predictor v2 (sac+logos+ava1-l14-linearMSE) on CLIP ViT-L/14 image
                           embeddings, 2 frames per second, mean
              videoscore2  VideoScore2 visual quality: greedy decoding at 2 fps, the expectation over levels 1-5
  stability   flicker      RAFT-large flow t->t+1 and t+1->t at 672x384, frame t+1 warped back to t; mean absolute
                           residual (0-255) over forward-backward consistent pixels after a Gaussian low-pass
                           (sigma 2 px), warp_l1_lp - lower is better. The unfiltered residual (warp_l1) rises with
                           sharpness and is reported only for reference.
              dino         DINOv2 ViT-S/14 CLS embeddings of all frames at 224x392: mean adjacent-frame cosine
  control     motion       CoTracker3 offline on the structure video and on the render, a 20x12 grid queried at
                           frames 0/40/80/120; error = |render track - structure track| / frame diagonal, over the
                           (point, frame) pairs where the structure track is reliable (visibility x confidence > 0.6)
              edges        Canny (5x5 blur, sigma 1.2, 50/150) on both, every 4th frame, edge F1 with a 2 px tolerance
  first frame firstframe   the DINOv2 embeddings against the first frame's: mean cosine over the last second
"""
from __future__ import annotations

import argparse
import json
import math
import os
import shutil
import subprocess
import sys
import time
import traceback
from pathlib import Path

import numpy as np
import torch
import torch.nn.functional as F

HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
import bench  # noqa: E402

MODELS = Path(os.environ.get("SOLAR_EVAL_MODELS", HERE.parent / "eval_models"))
FFMPEG = shutil.which("ffmpeg") or "ffmpeg"
ALL = ["dover", "laion_aes", "flicker", "dino", "firstframe", "motion", "edges"]
VERSION = "1"


# ---------------------------------------------------------------------------------------------- io

def decode(path: str, size: tuple[int, int] | None = None, fps: float | None = None) -> np.ndarray:
    """uint8 [T,H,W,3] RGB; size = (w, h), area-resampled."""
    vf = []
    if fps:
        vf.append(f"fps={fps}")
    if size:
        vf.append(f"scale={size[0]}:{size[1]}:flags=area")
    vf.append("format=rgb24")
    if size is None:
        w, h = probe_size(path)
    else:
        w, h = size
    raw = subprocess.run([FFMPEG, "-nostdin", "-loglevel", "error", "-i", path, "-vf", ",".join(vf),
                          "-f", "rawvideo", "-"], capture_output=True, check=True).stdout
    a = np.frombuffer(raw, np.uint8)
    n = len(a) // (w * h * 3)
    return a[: n * w * h * 3].reshape(n, h, w, 3)


def probe_size(path: str) -> tuple[int, int]:
    out = subprocess.run(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries", "stream=width,height",
                          "-of", "csv=p=0", path], capture_output=True, text=True).stdout.strip().split(",")
    return int(out[0]), int(out[1])


def load_png(path: str) -> np.ndarray:
    from PIL import Image
    return np.asarray(Image.open(path).convert("RGB"))


# ---------------------------------------------------------------------------------------------- models

class Models:
    """Lazily built, one per worker; weights from eval/setup.sh under SOLAR_EVAL_MODELS (default ./eval_models)."""

    def __init__(self, device):
        self.device = device
        self._m = {}

    def get(self, name):
        if name not in self._m:
            self._m[name] = getattr(self, f"_build_{name}")()
        return self._m[name]

    def _build_dover(self):
        import dover_score
        model, stypes, samplers = dover_score.build(self.device)
        return dover_score, model, stypes, samplers

    def _build_clip(self):
        from transformers import CLIPModel
        clip = CLIPModel.from_pretrained("openai/clip-vit-large-patch14", torch_dtype=torch.float32)
        clip = clip.to(self.device).eval()
        head = torch.nn.Sequential(  # improved-aesthetic-predictor MLP (simple_inference.py), dropouts are no-ops
            torch.nn.Linear(768, 1024), torch.nn.Dropout(0.2), torch.nn.Linear(1024, 128), torch.nn.Dropout(0.2),
            torch.nn.Linear(128, 64), torch.nn.Dropout(0.1), torch.nn.Linear(64, 16), torch.nn.Linear(16, 1))
        sd = torch.load(MODELS / "sac+logos+ava1-l14-linearMSE.pth", map_location="cpu")
        sd = {k.replace("layers.", ""): v for k, v in sd.items()}
        head.load_state_dict(sd, strict=True)
        return clip, head.to(self.device).eval()

    def _build_dino(self):
        import timm
        m = timm.create_model("vit_small_patch14_dinov2.lvd142m", pretrained=True, num_classes=0,
                              dynamic_img_size=True)
        return m.to(self.device).eval()

    def _build_raft(self):
        from torchvision.models.optical_flow import Raft_Large_Weights, raft_large
        return raft_large(weights=Raft_Large_Weights.C_T_SKHT_V2, progress=False).to(self.device).eval()

    def _build_cotracker(self):
        sys.path.insert(0, str(MODELS / "co-tracker"))
        from cotracker.models.build_cotracker import build_cotracker
        model = build_cotracker(None, offline=True, window_len=60, v2=False)
        sd = torch.load(MODELS / "cotracker3" / "baseline_offline.pth", map_location="cpu")
        if "model" in sd:
            sd = sd["model"]
        model.load_state_dict(sd, strict=True)
        return model.to(self.device).eval()

    def _build_vs2(self):
        import videoscore2
        model, processor, tokenizer = videoscore2.build()
        return videoscore2, model, processor, tokenizer


# ---------------------------------------------------------------------------------------------- metrics

@torch.no_grad()
def m_dover(path, M):
    ds, model, stypes, samplers = M.get("dover")
    raw = ds.score(path, model, stypes, samplers, M.device, 0)
    return {"dover": ds.fuse(raw), "dover_raw": raw}


CLIP_MEAN = torch.tensor([0.48145466, 0.4578275, 0.40821073]).view(1, 3, 1, 1)
CLIP_STD = torch.tensor([0.26862954, 0.26130258, 0.27577711]).view(1, 3, 1, 1)


def clip_pre(frames: np.ndarray) -> torch.Tensor:
    """openai CLIP preprocessing: bicubic resize of the short side to 224 (PIL), centre crop 224."""
    from PIL import Image
    out = []
    for f in frames:
        im = Image.fromarray(f)
        w, h = im.size
        s = 224 / min(w, h)
        im = im.resize((max(224, round(w * s)), max(224, round(h * s))), Image.BICUBIC)
        w, h = im.size
        l, t = (w - 224) // 2, (h - 224) // 2
        out.append(np.asarray(im.crop((l, t, l + 224, t + 224))))
    x = torch.from_numpy(np.stack(out)).permute(0, 3, 1, 2).float() / 255.0
    return (x - CLIP_MEAN) / CLIP_STD


@torch.no_grad()
def m_laion(frames24, M):
    clip, head = M.get("clip")
    sel = frames24[::12]  # 2 per second at 24 fps
    x = clip_pre(sel).to(M.device)
    emb = clip.visual_projection(clip.vision_model(pixel_values=x).pooler_output)
    emb = emb / emb.norm(dim=-1, keepdim=True)
    s = head(emb).squeeze(-1).float().cpu().numpy()
    return {"laion_aes": float(s.mean()), "laion_aes_min": float(s.min()), "frames": int(len(sel))}


IMNET_MEAN = torch.tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1)
IMNET_STD = torch.tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1)
DINO_HW = (224, 392)  # 16 x 28 patches of 14, the 1344x768 aspect (1.75)


@torch.no_grad()
def dino_embed(frames: np.ndarray, M) -> torch.Tensor:
    m = M.get("dino")
    out = []
    for i in range(0, len(frames), 64):
        x = torch.from_numpy(np.ascontiguousarray(frames[i:i + 64])).to(M.device).permute(0, 3, 1, 2).float() / 255
        x = F.interpolate(x, size=DINO_HW, mode="bicubic", align_corners=False, antialias=True).clamp(0, 1)
        x = (x - IMNET_MEAN.to(x.device)) / IMNET_STD.to(x.device)
        out.append(F.normalize(m(x).float(), dim=-1))
    return torch.cat(out)


def m_dino(emb):
    adj = (emb[1:] * emb[:-1]).sum(-1)
    return {"adjacent_cos": float(adj.mean()), "adjacent_cos_min": float(adj.min()),
            "first_last_cos": float((emb[0] * emb[-1]).sum())}


def m_firstframe(emb, first_emb, fps=24.0):
    cos = (emb @ first_emb).cpu().numpy().astype(np.float64)
    t = np.arange(len(cos)) / fps
    slope = float(np.polyfit(t, cos, 1)[0])
    return {"last1s_cos": float(cos[-int(fps):].mean()), "slope_per_s": slope, "frame0_cos": float(cos[0]),
            "mean_cos": float(cos.mean()), "curve_1fps": [round(float(c), 4) for c in cos[::int(fps)]]}


FLOW_WH = (672, 384)


@torch.no_grad()
def flows(frames: np.ndarray, M, batch=12):
    """Forward (t->t+1) and backward (t+1->t) RAFT flows at FLOW_WH. frames uint8 [T,h,w,3] already at FLOW_WH."""
    raft = M.get("raft")
    x = torch.from_numpy(np.ascontiguousarray(frames)).to(M.device).permute(0, 3, 1, 2).float() / 127.5 - 1.0
    fw, bw = [], []
    for i in range(0, len(x) - 1, batch):
        a = x[i:min(i + batch, len(x) - 1)]
        b = x[i + 1:min(i + batch, len(x) - 1) + 1]
        f = raft(torch.cat([a, b]), torch.cat([b, a]))[-1]
        fw.append(f[:len(a)]); bw.append(f[len(a):])
    return x, torch.cat(fw), torch.cat(bw)


def warp(img, flow):
    """Sample img at x + flow(x). img [N,C,H,W], flow [N,2,H,W] (pixels). Returns warped and in-bounds mask."""
    n, _, h, w = img.shape
    yy, xx = torch.meshgrid(torch.arange(h, device=img.device), torch.arange(w, device=img.device), indexing="ij")
    gx = xx[None] + flow[:, 0]
    gy = yy[None] + flow[:, 1]
    grid = torch.stack([2 * gx / (w - 1) - 1, 2 * gy / (h - 1) - 1], -1)
    out = F.grid_sample(img, grid, mode="bilinear", padding_mode="border", align_corners=True)
    inb = (gx >= 0) & (gx <= w - 1) & (gy >= 0) & (gy <= h - 1)
    return out, inb


@torch.no_grad()
def m_flicker(frames_lo: np.ndarray, M):
    x, fw, bw = flows(frames_lo, M)
    img_t, img_t1 = x[:-1], x[1:]
    warped, inb = warp(img_t1, fw)
    bw_at = warp(bw, fw)[0]  # backward flow sampled where the forward flow lands
    d = ((fw + bw_at) ** 2).sum(1)
    ok = inb & (d < 0.01 * ((fw ** 2).sum(1) + (bw_at ** 2).sum(1)) + 0.5)  # Sundaram et al. 2010 consistency check
    err = (img_t - warped).abs().mean(1) * 127.5  # [-1,1] -> 0-255 units
    okf = ok.float()
    per_pair = (err * okf).sum((1, 2)) / okf.sum((1, 2)).clamp(min=1)
    pp = per_pair.cpu().numpy()
    # low-pass residual: the warp residual blurred (Gaussian sigma 2 px at 672 wide, normalised convolution over the
    # consistent pixels) before |.|. Sub-pixel misalignment of fine texture is high-frequency and cancels; brightness /
    # colour flicker is low-frequency and stays. (The raw reading rises with sharpness.)
    k = torch.exp(-0.5 * (torch.arange(-6, 7, device=x.device).float() / 2.0) ** 2)
    k = (k / k.sum())

    def blur(z):  # [N,C,H,W], separable
        c = z.shape[1]
        z = F.conv2d(F.pad(z, (6, 6, 0, 0), mode="replicate"), k.view(1, 1, 1, -1).repeat(c, 1, 1, 1), groups=c)
        return F.conv2d(F.pad(z, (0, 0, 6, 6), mode="replicate"), k.view(1, 1, -1, 1).repeat(c, 1, 1, 1), groups=c)
    lp_pp = []
    for i in range(0, len(img_t), 16):
        m = okf[i:i + 16, None]
        r = (img_t[i:i + 16] - warped[i:i + 16]) * m
        bm = blur(m)
        lp = (blur(r) / bm.clamp(min=1e-3)).abs().mean(1) * 127.5
        v = m[:, 0] * (bm[:, 0] > 0.5)
        lp_pp.append((lp * v).sum((1, 2)) / v.sum((1, 2)).clamp(min=1))
    lp_pp = torch.cat(lp_pp).cpu().numpy()
    g = (0.299 * x[:, 0] + 0.587 * x[:, 1] + 0.114 * x[:, 2]) * 127.5  # grey, 0-255 offsets
    lap = -4 * g[:, 1:-1, 1:-1] + g[:, :-2, 1:-1] + g[:, 2:, 1:-1] + g[:, 1:-1, :-2] + g[:, 1:-1, 2:]
    return {"warp_l1": float(pp.mean()), "warp_l1_p90": float(np.percentile(pp, 90)),
            "warp_l1_lp": float(lp_pp.mean()), "warp_l1_lp_p90": float(np.percentile(lp_pp, 90)),
            "sharpness_lapvar": float(lap.flatten(1).var(1).mean()),
            "warp_l1_max": float(pp.max()), "consistent_frac": float(okf.mean()),
            "flow_mag": float(fw.norm(dim=1).mean()), "pairs": int(len(pp))}


GRID = (20, 12)  # x, y
QUERY_FRAMES = (0, 40, 80, 120)  # a fresh grid every 40 frames: under fast camera moves a frame-0 grid has left the
                                 # picture by frame 60 (0-22% of frame-0 points were still in frame at 100)
CT_HW = (384, 512)
REL_THR = 0.6


@torch.no_grad()
def track(frames: np.ndarray, M, query_frames=QUERY_FRAMES):
    """CoTracker3 offline; a GRID at each query frame. Returns tracks [T,N,2] (input pixels), vis, conf [T,N],
    query xy [N,2], query frame [N]."""
    model = M.get("cotracker")
    T, H, W, _ = frames.shape
    v = torch.from_numpy(frames.copy()).to(M.device).permute(0, 3, 1, 2).float()
    v = F.interpolate(v, CT_HW, mode="bilinear", align_corners=True)[None]  # as CoTrackerPredictor does
    gx = (np.arange(GRID[0]) + 0.5) / GRID[0] * W
    gy = (np.arange(GRID[1]) + 0.5) / GRID[1] * H
    grid = np.array([(x, y) for y in gy for x in gx], np.float32)
    qf = [t for t in query_frames if t < T - 1]
    pts = np.concatenate([grid] * len(qf))
    qt = np.repeat(np.array(qf), len(grid))
    q = torch.zeros(1, len(pts), 3, device=M.device)
    q[0, :, 0] = torch.from_numpy(qt).to(M.device).float()
    q[0, :, 1] = torch.from_numpy(pts[:, 0]).to(M.device) * (CT_HW[1] - 1) / (W - 1)
    q[0, :, 2] = torch.from_numpy(pts[:, 1]).to(M.device) * (CT_HW[0] - 1) / (H - 1)
    tr, vis, conf, _ = model(video=v, queries=q, iters=6)
    tr = tr[0] * tr.new_tensor([(W - 1) / (CT_HW[1] - 1), (H - 1) / (CT_HW[0] - 1)])
    tr[torch.from_numpy(qt).long(), torch.arange(len(pts))] = torch.from_numpy(pts).to(tr.device)  # query = query
    return tr.float().cpu().numpy(), vis[0].float().cpu().numpy(), conf[0].float().cpu().numpy(), pts, qt


def motion_compare(tc, vc, cc, tm, vm, cm, pts, qt, W, H):
    """Per (point, frame after its query frame) where the cond track is reliable."""
    diag = math.hypot(W, H)
    n = min(len(tc), len(tm))
    t = np.arange(n)[:, None]
    after = t > qt[None]
    rel = ((vc[:n] * cc[:n]) > REL_THR) & after
    err = np.linalg.norm(tm[:n] - tc[:n], axis=-1) / diag
    still = np.linalg.norm(tc[:n] - pts[None], axis=-1) / diag  # a frozen render: every track stays at its query
    out_vis = (vm[:n] * cm[:n]) > REL_THR
    # velocity step k -> k+1, valid when both ends are reliable (the query frame end is exact)
    start_ok = ((vc[:n - 1] * cc[:n - 1]) > REL_THR) & (t[:-1] > qt[None]) | (t[:-1] == qt[None])
    vrel = rel[1:] & start_ok
    verr = np.linalg.norm((tm[1:n] - tm[:n - 1]) - (tc[1:n] - tc[:n - 1]), axis=-1) / diag
    if rel.sum() == 0:
        return {"kept_frac": 0.0, "err_mean": None}
    return {"err_mean": float(err[rel].mean()), "frac_lt_2pct": float((err[rel] < 0.02).mean()),
            "err_median": float(np.median(err[rel])),
            "vel_err_mean": float(verr[vrel].mean()) if vrel.any() else None,
            "frozen_err_mean": float(still[rel].mean()), "frozen_frac_lt_2pct": float((still[rel] < 0.02).mean()),
            "kept_frac": float(rel.sum() / after.sum()), "kept_pairs": int(rel.sum()),
            "kept_frac_by_query": [round(float(rel[:, qt == q].sum() / max(after[:, qt == q].sum(), 1)), 3)
                                   for q in np.unique(qt)],
            "render_reliable_frac": float(out_vis[rel].mean()), "frames": int(n)}


def canny(frames: np.ndarray) -> np.ndarray:
    import cv2
    return np.stack([cv2.Canny(cv2.GaussianBlur(cv2.cvtColor(f, cv2.COLOR_RGB2GRAY), (5, 5), 1.2), 50, 150)
                     for f in frames])


def edge_errors(made: np.ndarray, want: np.ndarray, tolerance: int = 2, level: int = 64) -> dict:
    """Edge F1: an edge counts as found when the other side has one within ``tolerance`` pixels."""

    frames = min(len(made), len(want))
    found = torch.from_numpy(np.ascontiguousarray(made[:frames]) >= level).float()[:, None]
    given = torch.from_numpy(np.ascontiguousarray(want[:frames]) >= level).float()[:, None]
    window = 2 * tolerance + 1

    def near(edges: torch.Tensor) -> torch.Tensor:
        return torch.nn.functional.max_pool2d(edges, window, stride=1, padding=tolerance)

    hits_found = float((found * near(given)).sum())
    hits_given = float((given * near(found)).sum())
    precision = hits_found / max(float(found.sum()), 1.0)
    recall = hits_given / max(float(given.sum()), 1.0)
    f1 = 2 * precision * recall / (precision + recall) if precision + recall > 0 else 0.0
    return {"F1": f1, "precision": precision, "recall": recall, "frames": frames}


def m_edges(render_full, cond_full):
    r, c = render_full[::4], cond_full[::4]
    n = min(len(r), len(c))
    out = edge_errors(canny(r[:n]), canny(c[:n]))
    out["cond_edge_density"] = float((canny(c[:n]) > 0).mean())
    return out


@torch.no_grad()
def m_vs2(path, prompt, M):
    vs, model, processor, tokenizer = M.get("vs2")
    r = vs.judge(model, processor, tokenizer, path, prompt)
    return {"visual_quality": r["visual_quality"], "visual_quality_expected": r["visual_quality_expected"],
            "alignment": r["alignment"], "physics": r["physics"], "reason": r["reason"][-400:]}


# ---------------------------------------------------------------------------------------------- driver

def compute(need: list[str], render: str, cond: str | None, first: str | None, M, prompt: str = "") -> dict:
    """All metrics in need on local files; returns {metric: payload with 'sec'}."""
    res = {}

    def timed(name, fn):
        torch.cuda.synchronize(M.device)
        t0 = time.time()
        try:
            out = fn()
        except Exception:
            traceback.print_exc()
            out = {"error": traceback.format_exc()[-400:]}
        torch.cuda.synchronize(M.device)
        out["sec"] = round(time.time() - t0, 2)
        res[name] = out

    full = decode(render) if set(need) & {"laion_aes", "dino", "firstframe", "motion", "edges"} else None
    if full is not None and len(full) == 0:
        raise RuntimeError("render decodes to 0 frames")
    cv = None
    if set(need) & {"motion", "edges"}:
        cv = decode(cond)
        if cv.shape[1:3] != full.shape[1:3]:
            cv = decode(cond, size=(full.shape[2], full.shape[1]))
    if "dover" in need:
        timed("dover", lambda: m_dover(render, M))
    if "laion_aes" in need:
        timed("laion_aes", lambda: m_laion(full, M))
    if "flicker" in need:
        timed("flicker", lambda: m_flicker(decode(render, size=FLOW_WH), M))
    if "dino" in need or "firstframe" in need:
        t0 = time.time()
        emb = dino_embed(full, M)
        te = time.time() - t0
        if "dino" in need:
            timed("dino", lambda: m_dino(emb))
            res["dino"]["sec"] = round(res["dino"]["sec"] + te, 2)
        if "firstframe" in need:
            timed("firstframe", lambda: m_firstframe(emb, dino_embed(load_png(first)[None], M)[0]))
            res["firstframe"]["sec"] = round(res["firstframe"]["sec"] + te, 2)
    if "motion" in need:
        def mo():
            # the query frames both videos have: a render shorter than its structure (Wan, 121 frames) would
            # otherwise track a different number of points than the structure
            qf = tuple(t for t in QUERY_FRAMES if t < min(len(cv), len(full)) - 1)
            tc, vc, cc, pts, qt = track(cv, M, qf)
            tm, vm, cm, _, _ = track(full, M, qf)
            return motion_compare(tc, vc, cc, tm, vm, cm, pts, qt, full.shape[2], full.shape[1])
        timed("motion", mo)
    if "edges" in need:
        timed("edges", lambda: m_edges(full, cv))
    if "videoscore2" in need:
        timed("videoscore2", lambda: m_vs2(render, prompt, M))
    return res


def result_path(out: str, metric: str, method: str, case: bench.Case) -> Path:
    """<out>/<metric>/<method>/hard/<case_id>.json or <out>/<metric>/<method>/heldout/<clip_id>.json"""
    return Path(out) / metric / method / case.split / f"{case.key}.json"


def main():
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--bench", required=True, help="local copy of the SolarRenderer-Data dataset")
    ap.add_argument("--renders", required=True, help="<renders>/hard/<case_id>.mp4, <renders>/heldout/<clip_id>.mp4")
    ap.add_argument("--method", required=True, help="the name results are filed under")
    ap.add_argument("--out", required=True)
    ap.add_argument("--split", default="all", choices=("hard", "heldout", "all"))
    ap.add_argument("--metrics", default=",".join(ALL), help=f"comma list of {ALL + ['videoscore2']}")
    ap.add_argument("--shard", type=int, default=0)
    ap.add_argument("--shards", type=int, default=1)
    ap.add_argument("--device", default="cuda:0")
    a = ap.parse_args()
    metrics = [m for m in a.metrics.split(",") if m]
    M = Models(torch.device(a.device))
    torch.cuda.set_device(M.device)
    cases = bench.load(a.bench, a.split)[a.shard::a.shards]
    done = skipped = 0
    for c in cases:
        render = c.render(a.renders)
        need = [m for m in metrics if not result_path(a.out, m, a.method, c).exists()]
        if not need:
            continue
        if not render.exists():
            skipped += 1
            continue
        try:
            res = compute(need, str(render), c.condition, c.first_frame, M, c.prompt)
        except Exception:
            traceback.print_exc()
            print(f"SKIP {c.key}", flush=True)
            continue
        for m, payload in res.items():
            if "error" in payload:  # not written: the next run retries it
                print(f"{c.key} {m} ERROR {payload['error'][-200:]}", flush=True)
                continue
            p = result_path(a.out, m, a.method, c)
            p.parent.mkdir(parents=True, exist_ok=True)
            rec = {"id": c.id, "split": c.split, "method": a.method, "metric": m, "kind": c.kind,
                   "tags": c.tags, "version": VERSION, **payload}
            p.write_text(json.dumps(rec, ensure_ascii=False))
        done += 1
        brief = {m: {k: (round(v, 4) if isinstance(v, float) else v) for k, v in p.items()
                     if not isinstance(v, (list, dict, str))} for m, p in res.items()}
        print(f"{c.key} {json.dumps(brief)}", flush=True)
    print(f"done: {done} cases scored, {skipped} without a render", flush=True)


if __name__ == "__main__":
    main()