Skip to content

Official Codebase of 3DCodeVerse

Python
11.8 KB
Raw Blame
"""Score one gen dir of one suite → <gen_dir>/metrics.jsonl + summary.json (one schema for every suite).

summary.json
  suite, dialect, model, n, statuses{OK,EMPTY,FAIL,CRASH,TIMEOUT,MISSING}, exec_ok, exec_rate,
  n_ref_ok (tasks whose reference produced a mesh — the denominator of the *_all geometry means),
  n_scored, chamfer_mean_scored, chamfer_median_scored, f05_mean_scored, f10_mean_scored, iou_mean_scored,
  f05_mean_all, f10_mean_all, iou_mean_all          (fail = 0, over n_ref_ok)
  render_ok, render_rate                            (glsl: compiled AND painted a non-uniform image)
  truncated, mean_new_tokens, extract{no_fence, multi_block, chosen_not_last, from_think_fallback, ...}
  gen{backend, model, temperature, seed, max_new_tokens, no_think}
"""
from __future__ import annotations

import json
from collections import Counter

import numpy as np
from concurrent.futures import ProcessPoolExecutor
from pathlib import Path

from . import metrics
from ._jsonl import read_rows
from .suites import get_suite, load_prompts, resolve


def _geom(args):
    gen_mesh, ref_mesh, n_points = args
    try:
        return metrics.geometry_metrics(gen_mesh, ref_mesh, n_points=n_points)
    except Exception as e:  # noqa: BLE001
        return {"metric_error": f"{type(e).__name__}: {str(e)[:200]}"}


def _geom_official(args):
    gen_mesh, ref_mesh = args
    try:
        return metrics.chamfer_official(gen_mesh, ref_mesh)
    except Exception as e:  # noqa: BLE001
        return {"metric_error_official": f"{type(e).__name__}: {str(e)[:200]}"}


STRUCT_FIELDS = ("n_components", "floating_part_rate", "floating_area_fraction", "largest_component_volume_fraction",
                 "nonmanifold_edge_rate", "boundary_edge_rate", "is_watertight", "ground_contact", "com_over_support",
                 "com_support_margin")
STRUCT_PENALTY = {"floating_part_rate": 1.0, "floating_area_fraction": 1.0, "largest_component_volume_fraction": 0.0,
                  "nonmanifold_edge_rate": 1.0, "boundary_edge_rate": 1.0, "is_watertight": 0.0, "ground_contact": 0.0,
                  "com_over_support": 0.0}


def _struct(args):
    """Official structural-integrity fields (structural.py) for one generated mesh (+ component ratio vs the reference)."""
    gen_mesh, ref_mesh = args
    from . import structural

    up = 1 if str(gen_mesh).endswith(".glb") else 2
    try:
        rec, st = structural.score_glb(gen_mesh, 0.01, up, 1e-3, 100_000)
        if rec is None:
            return {"struct_status": st}
        out = {"struct_status": st, **{k: rec.get(k) for k in STRUCT_FIELDS}}
        if ref_mesh:
            rrec, rst = structural.score_glb(ref_mesh, 0.01, up, 1e-3, 100_000)
            if rrec:
                out["ref_n_components"] = rrec["n_components"]
                out["abs_log2_component_ratio"] = abs(float(np.log2(max(rec["n_components"] / max(rrec["n_components"], 1), 1e-9))))
        return out
    except Exception as e:  # noqa: BLE001
        return {"struct_status": f"SCORE_FAIL: {type(e).__name__}: {str(e)[:120]}"}


def score_dir(gen_dir: Path, suite: str, ref: str = "executed", workers: int = 4, with_geometry: bool = True,
              image_encoders: tuple[str, ...] = ("siglip2", "dinov3"), device: str = "cuda") -> dict:
    gen_dir = Path(gen_dir)
    spec = get_suite(suite)
    rows = {r["id"]: r for r in load_prompts(suite)}
    ex = {}
    if (gen_dir / "exec_results.jsonl").exists():
        ex = {r["id"]: r for r in read_rows(gen_dir / "exec_results.jsonl")}
    gens = {}
    if (gen_dir / "gens.jsonl").exists():
        gens = {r["id"]: r for r in read_rows(gen_dir / "gens.jsonl")}
    gen_stats = json.loads((gen_dir / "gen_stats.json").read_text()) if (gen_dir / "gen_stats.json").exists() else {}
    n_prompts_total = len(rows)
    if gens:   # a --limit run: score only what was generated, but say so in the summary
        rows = {tid: r for tid, r in rows.items() if tid in gens}

    out_rows, geom_jobs, struct_jobs = [], [], []
    for tid, row in rows.items():
        e = ex.get(tid, {})
        r = {"id": tid, "status": e.get("status", "MISSING"), "error": (e.get("error") or None) and str(e.get("error"))[:200],
             "latency_s": e.get("latency_s"), "n_new_tokens": gens.get(tid, {}).get("n_new_tokens"),
             "truncated": gens.get(tid, {}).get("truncated")}
        if "render" in spec.metrics:
            r["render"] = e.get("render")
        ref_rel = row["reference"].get("mesh")
        if ref == "canonical" and ref_rel:
            cand = resolve(ref_rel).with_name("ref_canonical.glb")
            ref_rel = str(cand.relative_to(resolve("."))) if cand.exists() else ref_rel
        ref_mesh = resolve(ref_rel) if ref_rel else None
        r["has_ref"] = bool(ref_mesh and ref_mesh.exists())
        if with_geometry and "geometry" in spec.metrics and r["status"] == "OK" and e.get("mesh") and r["has_ref"]:
            geom_jobs.append((len(out_rows), (e["mesh"], str(ref_mesh), spec.n_points)))
        if with_geometry and spec.dialect in ("blender", "cadquery", "openscad") and r["status"] == "OK" and e.get("mesh"):
            struct_jobs.append((len(out_rows), (e["mesh"], str(ref_mesh) if r["has_ref"] else None)))
        out_rows.append(r)

    if geom_jobs:
        with ProcessPoolExecutor(workers) as pool:
            for (i, _), res in zip(geom_jobs, pool.map(_geom, [j for _, j in geom_jobs])):
                out_rows[i].update(res)
            if "chamfer_official" in spec.metrics:
                for (i, _), res in zip(geom_jobs, pool.map(_geom_official, [(j[0], j[1]) for _, j in geom_jobs])):
                    out_rows[i].update(res)

    if struct_jobs:
        with ProcessPoolExecutor(workers) as pool:
            for (i, _), res in zip(struct_jobs, pool.map(_struct, [j for _, j in struct_jobs])):
                out_rows[i].update(res)

    image_summ = {}
    if "image_sim" in spec.metrics and any(r["reference"].get("renders") for r in rows.values()):
        rendered = {p.parent.parent.parent.name for p in gen_dir.glob("*/exec/renders/Image_035.png")}
        for tid in rows:
            out_rows[[r["id"] for r in out_rows].index(tid)]["rendered"] = tid in rendered
        if rendered:
            from . import image_sim
            for enc in image_encoders:
                try:
                    image_summ[enc] = image_sim.score_gen_dir(gen_dir, rows, resolve, enc, device, text_sim=(enc.startswith("siglip") and not any(r.get("images") for r in rows.values())))
                    per = {r["id"]: r for r in read_rows(gen_dir / f"image_sim_{enc}.jsonl")}
                    for r in out_rows:
                        p = per.get(r["id"], {})
                        r[f"{enc}_paired"], r[f"{enc}_assigned"] = p.get("view_paired"), p.get("best_assignment")
                        if "text_image" in p:
                            r[f"{enc}_text"] = p["text_image"]
                except Exception as e:  # noqa: BLE001
                    image_summ[enc] = {"error": f"{type(e).__name__}: {str(e)[:200]}"}
                    print(f"[score] image_sim {enc} failed: {image_summ[enc]['error']}", flush=True)

    with (gen_dir / "metrics.jsonl").open("w") as f:
        for r in out_rows:
            f.write(json.dumps(r) + "\n")

    n = len(out_rows)
    st = Counter(r["status"] for r in out_rows)
    ok = st.get("OK", 0)
    summ = {"suite": suite, "dialect": spec.dialect, "model": gen_stats.get("model"), "n": n, "n_prompts_total": n_prompts_total,
            "statuses": dict(st),
            "exec_ok": ok, "exec_rate": ok / n if n else None,
            "truncated": sum(bool(r.get("truncated")) for r in out_rows),
            "mean_new_tokens": metrics.mean([r.get("n_new_tokens") for r in out_rows]),
            "extract": {k: gen_stats.get(k) for k in ("no_fence", "multi_block", "chosen_not_last", "from_think_fallback",
                                                       "syntax_valid", "no_dialect_marker", "with_think")},
            "gen": {k: gen_stats.get(k) for k in ("backend", "model", "temperature", "seed", "max_new_tokens", "no_think",
                                                   "tok_per_s", "wall_s")}}
    if "geometry" in spec.metrics:
        with_ref = [r for r in out_rows if r["has_ref"]]
        scored = [r for r in with_ref if "chamfer" in r]
        n_ref = len(with_ref)
        summ.update(n_ref_ok=n_ref, n_scored=len(scored),
                    chamfer_mean_scored=metrics.mean([r["chamfer"] for r in scored]),
                    chamfer_median_scored=metrics.median([r["chamfer"] for r in scored]),
                    f05_mean_scored=metrics.mean([r["[email protected]"] for r in scored]),
                    f10_mean_scored=metrics.mean([r["[email protected]"] for r in scored]),
                    iou_mean_scored=metrics.mean([r.get("iou_vox") for r in scored]),
                    f05_mean_all=(sum(r["[email protected]"] for r in scored) / n_ref) if n_ref else None,
                    f10_mean_all=(sum(r["[email protected]"] for r in scored) / n_ref) if n_ref else None,
                    iou_mean_all=(sum(r.get("iou_vox") or 0 for r in scored) / n_ref) if n_ref else None)
    if struct_jobs:
        sc = [r for r in out_rows if r.get("struct_status") == "OK"]
        summ["structural"] = {"n_scored": len(sc)}
        for k, pen in STRUCT_PENALTY.items():
            vals = [float(r[k]) for r in sc if r.get(k) is not None]
            summ["structural"][f"{k}_cond"] = metrics.mean(vals)
            summ["structural"][f"{k}_pen"] = (sum(vals) + pen * (n - len(vals))) / n if n else None
        summ["structural"]["n_components_median_cond"] = metrics.median([r.get("n_components") for r in sc])
        summ["structural"]["abs_log2_component_ratio_cond"] = metrics.mean([r.get("abs_log2_component_ratio") for r in sc])
    if "chamfer_official" in spec.metrics:
        sc = [r for r in out_rows if r.get("cd_official_yawmin") is not None]
        pen = 1.5 * max((r["cd_official_yawmin"] for r in sc), default=0.0)   # official penalty: 1.5 x worst observed
        summ.update(cd_official_cond=metrics.mean([r["cd_official"] for r in sc]),
                    cd_official_yawmin_cond=metrics.mean([r["cd_official_yawmin"] for r in sc]),
                    cd_official_yawmin_pen=(sum(r.get("cd_official_yawmin") if r.get("cd_official_yawmin") is not None else pen for r in out_rows) / n) if n else None)
    if image_summ:
        summ["image_sim"] = image_summ
    if "render" in spec.metrics:
        rok = sum(r.get("render") == "OK" for r in out_rows)
        summ.update(render_ok=rok, render_rate=rok / n if n else None,
                    render_static=sum(r.get("render") == "STATIC" for r in out_rows))
    (gen_dir / "summary.json").write_text(json.dumps(summ, indent=2))
    line = f"[score] {suite} {summ.get('model')}: exec {ok}/{n} = {100 * (summ['exec_rate'] or 0):.1f}%"
    if "geometry" in spec.metrics:
        line += f" | [email protected] all {summ['f05_mean_all'] or 0:.3f} (ok {summ['f05_mean_scored'] or 0:.3f}) | CD {summ['chamfer_mean_scored'] or 0:.3f} | IoU {summ['iou_mean_scored'] or 0:.3f}"
    if "render" in spec.metrics:
        line += f" | render OK {summ['render_ok']}/{n}"
    if struct_jobs and summ.get("structural", {}).get("floating_part_rate_cond") is not None:
        st_ = summ["structural"]
        line += f" | floating {st_['floating_part_rate_cond']:.3f} | watertight {st_['is_watertight_cond']:.2f} | stable {st_['com_over_support_cond']:.2f}"
    for enc, s in image_summ.items():
        if "view_paired" in s:
            line += f" | {enc} paired {s['view_paired']['penalized'] or 0:.3f} (cond {s['view_paired']['conditional'] or 0:.3f})"
    print(line, flush=True)
    return summ


if __name__ == "__main__":
    import argparse

    ap = argparse.ArgumentParser()
    ap.add_argument("gen_dir", type=Path)
    ap.add_argument("--suite", required=True)
    ap.add_argument("--ref", choices=["executed", "canonical"], default="executed")
    ap.add_argument("--workers", type=int, default=8)
    a = ap.parse_args()
    score_dir(a.gen_dir, a.suite, a.ref, a.workers)