Skip to content

Official Codebase of 3DCodeVerse

Python

flywheel/gallery/skills: delete test-only and never-passed surface; quote gallery query links, fix unique_files root and the -dirty scope

gaoypengfe2f0e4ede8c
diff --git a/harness/codeverse/flywheel/export.py b/harness/codeverse/flywheel/export.py
index 7dde56fe..5fe6a4ab 100644
--- a/harness/codeverse/flywheel/export.py
+++ b/harness/codeverse/flywheel/export.py
@@ -108,7 +108,7 @@ class ExportedSample(NamedTuple):
def export_one(
- ws: Workspace, record: RunRecord, out_dir: Path, *, overwrite: bool = True,
+ ws: Workspace, record: RunRecord, out_dir: Path, *,
captions_dir: Path | None = None, run_id: RunId | None = None
) -> ExportedSample:
"""Write the sample folder for one run; returns the dir plus a sha256 per file
@@ -123,8 +123,6 @@ def export_one(
if not files:
raise S.SampleError("no code files in src/ (nothing to export)")
if dest.exists():
- if not overwrite:
- raise S.SampleError(f"sample dir exists: {dest}")
shutil.rmtree(dest)
dest.mkdir(parents=True)
entry, written = S.write_code_tree(dest, files, record.spec.language)
@@ -154,16 +152,13 @@ def export_samples(
*,
min_score: float | None = None,
only_passed: bool = False,
- best_round: bool = True,
- overwrite: bool = True,
include_unbuilt: bool = False,
captions_dir: Path | str | None = None,
drop_duplicates: bool = False,
) -> ExportReport:
"""Export every eligible run under ``runs_dir`` into ``out_dir`` and rebuild the index.
- ``best_round`` is the only supported selection (kept as a parameter for
- API stability); the best round is chosen by ``record.best_round`` or the
+ The best round is chosen by ``record.best_round`` or the
highest judged score. Runs whose best round never built are skipped unless
``include_unbuilt`` (failed runs are still useful for repair pairs, not as
dataset samples). Byte-identical duplicates (same raw ``code_sha256`` + prompt)
@@ -173,8 +168,6 @@ def export_samples(
MARKED (``near_duplicate_of``) — never dropped: whitespace inside a string
literal is not a duplicate.
"""
- if not best_round:
- raise ValueError("export_samples: only best_round=True is supported")
out = Path(out_dir)
out.mkdir(parents=True, exist_ok=True)
rep = ExportReport()
@@ -221,7 +214,7 @@ def export_samples(
_skip(ws.root, rid, f"score {score} < min_score {min_score}")
continue
try:
- exported = export_one(ws, rec, out, overwrite=overwrite,
+ exported = export_one(ws, rec, out,
captions_dir=Path(captions_dir) if captions_dir else None, run_id=rid)
except S.SampleError as e:
_skip(ws.root, rid, str(e))
@@ -256,7 +249,7 @@ def export_samples(
rep.manifest = str(write_manifest(DatasetGenerationManifest(
filters=ManifestFilters(min_score=min_score, only_passed=only_passed,
include_unbuilt=include_unbuilt, drop_duplicates=drop_duplicates,
- best_round=best_round, captions_dir=str(captions_dir or "")),
+ captions_dir=str(captions_dir or "")),
entries=entries, dropped=dropped), out))
rep.n_indexed = len(rows)
for r in rows:
@@ -320,20 +313,6 @@ def row_for_sample(sample_dir: Path) -> dict[str, Any]:
}
-def collect_rows(out_dir: Path) -> list[dict[str, Any]]:
- """Recovery/debug rescan of a dataset folder — export/pack build their rows from
- the manifest, never from this. Exactly three levels deep (track/language/key,
- the shape ``sample_rel_dir`` has), so an LLM-written ``src/meta.json`` inside a
- sample's code tree can never inject a phantom row."""
- rows = []
- for meta in sorted(out_dir.glob("*/*/*/meta.json")):
- try:
- rows.append(row_for_sample(meta.parent))
- except (OSError, ValueError, KeyError) as e:
- log.warning("skipping %s: %s", meta.parent, e)
- return rows
-
-
def parquet_schema() -> Any:
import pyarrow as pa
diff --git a/harness/codeverse/flywheel/record.py b/harness/codeverse/flywheel/record.py
index d6f092f0..70b308e0 100644
--- a/harness/codeverse/flywheel/record.py
+++ b/harness/codeverse/flywheel/record.py
@@ -71,7 +71,7 @@ def _harness_git_sha(pkg_file: Path | None = None) -> str:
return ""
sha = head.stdout.strip()
# dirty = any tracked file under the package tree (harness/) modified; untracked files ignored
- pkg_root = here.parents[1] if here.parent.name == "flywheel" else here.parent
+ pkg_root = here.parents[2] if here.parent.name == "flywheel" else here.parent
status = _git(["status", "--porcelain", "--untracked-files=no", "--", str(pkg_root)], here.parent)
if status is not None and status.returncode == 0 and status.stdout.strip():
sha += "-dirty"
@@ -328,8 +328,9 @@ def unique_files(root: Path | str, name: str) -> list[Path]:
skipped. The one walker for every survey that counts sessions, records or artefacts —
each hand-rolled copy of it has at some point counted a file twice."""
seen: dict[Path, Path] = {}
- for p in sorted(Path(root).rglob(name, recurse_symlinks=True)):
- if SUBRUN_DIRS & set(p.parts):
+ root = Path(root)
+ for p in sorted(root.rglob(name, recurse_symlinks=True)):
+ if SUBRUN_DIRS & set(p.relative_to(root).parts):
continue
seen.setdefault(p.resolve(), p)
return sorted(seen.values())
diff --git a/harness/codeverse/gallery/compare.py b/harness/codeverse/gallery/compare.py
index 0b7a06b5..e2895f59 100644
--- a/harness/codeverse/gallery/compare.py
+++ b/harness/codeverse/gallery/compare.py
@@ -13,6 +13,7 @@ from __future__ import annotations
import csv
import io
+from urllib.parse import quote
from codeverse.gallery.cards import fmt, tier_tag, verdict_tag
from codeverse.gallery.model import VERDICT_META, RunEntry
@@ -137,7 +138,7 @@ def render_compare(entries: list[RunEntry], urls: UrlMaker, *, note: str = "") -
body = (
top_bar("3dcv gallery", f"{len(entries)} runs · {tally}",
crumbs="<a href='/'>gallery</a> <span class='faint'>/</span> <b>compare</b>",
- right=f"<a class='btn' href='/export.csv?runs={esc(','.join(e.key for e in entries))}'>"
+ right=f"<a class='btn' href='/export.csv?runs={quote(','.join(e.key for e in entries))}'>"
f"export csv</a>")
+ "<main class='wrap'>"
+ (f"<p class='small muted'>{esc(note)}</p>" if note else "")
diff --git a/harness/codeverse/gallery/detail.py b/harness/codeverse/gallery/detail.py
index 9fe6d6a6..c4e3c777 100644
--- a/harness/codeverse/gallery/detail.py
+++ b/harness/codeverse/gallery/detail.py
@@ -10,13 +10,14 @@ bench is still writing shows its latest rounds without a restart.
from __future__ import annotations
from pathlib import Path
+from urllib.parse import quote
from codeverse.contracts.common import ENTRY_FILE
from codeverse.contracts.run import RunRecord
from codeverse.flywheel.record import best_round_record, effective_judgment
from codeverse.gallery.cards import fmt, gallery_figure, tier_tag, verdict_tag
from codeverse.gallery.code import CODE_CSS, numbered, read_text, src_files
-from codeverse.gallery.index import _rel, best_sheet, hero_view
+from codeverse.gallery.index import _rel
from codeverse.gallery.model import RunEntry
from codeverse.gallery.theme import esc, footer, page_shell, top_bar
from codeverse.gallery.urls import UrlMaker
@@ -351,17 +352,12 @@ def render_detail(entry: RunEntry, urls: UrlMaker, ws: Workspace, rec: RunRecord
prev: RunEntry | None = None, nxt: RunEntry | None = None) -> str:
"""Full detail page for a run whose record parsed."""
best = entry.best_round
- if not entry.sheet:
- entry = entry.model_copy(update={"sheet": best_sheet(ws, rec)})
- if not entry.hero:
- hero, label, n = hero_view(ws, rec)
- entry = entry.model_copy(update={"hero": hero, "hero_label": label, "n_views": n})
nav = "".join(f"<a href='#{a}'>{a}</a>" for a in
("rounds", "judge", "complexity", "measurement", "renders", "cost", "code"))
body = (
top_bar("3dcv gallery",
crumbs=f"<a href='/'>gallery</a> <span class='faint'>/</span> "
- f"<a href='/?battery={esc(entry.battery)}'>{esc(entry.battery)}</a> "
+ f"<a href='/?battery={quote(entry.battery)}'>{esc(entry.battery)}</a> "
f"<span class='faint'>/</span> <b>{esc(entry.slug)}</b>",
right=_neighbours(urls, prev, nxt) if urls.has_detail else "")
+ "<main class='wrap'>"
diff --git a/harness/codeverse/gallery/index.py b/harness/codeverse/gallery/index.py
index c09571a8..5dbf7097 100644
--- a/harness/codeverse/gallery/index.py
+++ b/harness/codeverse/gallery/index.py
@@ -54,11 +54,6 @@ def default_roots(base: Path | str = ".") -> list[Path]:
return roots
-def root_label(root: Path) -> str:
- """Section label: ``runs`` for the plain runs dir, the battery name for a bench out dir."""
- return battery_label(Path(root))
-
-
def _rel(ws: Workspace, path: Path | str | None) -> str:
"""``path`` (rebased into this workspace) as a run-relative posix string, or ``""``
when it is outside the run."""
@@ -320,7 +315,7 @@ def scan_root(root: Path, label: str | None = None) -> RootSection:
that still repeats within the root gets ``#2``/``#3`` — the same precedent as
duplicate section labels in :func:`build_index`."""
root = Path(root)
- label = label or root_label(root)
+ label = label or battery_label(root)
found = find_run_dirs(root, predicate=is_run_dir)
dirs = sorted({*found, *_pending_direct_children(root, set(found))})
used: dict[str, int] = {}
@@ -341,7 +336,7 @@ def build_index(roots: list[Path] | list[str]) -> GalleryIndex:
used: dict[str, int] = {}
for root in roots:
root = Path(root)
- base = root_label(root)
+ base = battery_label(root)
used[base] = used.get(base, 0) + 1
label = base if used[base] == 1 else f"{base}#{used[base]}"
sections.append(scan_root(root, label))
diff --git a/harness/codeverse/gallery/model.py b/harness/codeverse/gallery/model.py
index 0b358e57..c9008d23 100644
--- a/harness/codeverse/gallery/model.py
+++ b/harness/codeverse/gallery/model.py
@@ -238,10 +238,6 @@ class Summary(BaseModel):
minutes: float = 0.0
broken: int = 0
- def sums(self) -> bool:
- """The invariant the strip promises the reader."""
- return sum(self.breakdown.values()) == self.n
-
def verdict_breakdown(entries: list[RunEntry]) -> dict[str, int]:
"""``{bucket: count}`` over all four buckets (zeros included), summing to ``len(entries)``."""
diff --git a/harness/codeverse/reference.py b/harness/codeverse/reference.py
index 2cc2f2b2..eccd3607 100644
--- a/harness/codeverse/reference.py
+++ b/harness/codeverse/reference.py
@@ -210,7 +210,7 @@ class ReferenceDiff(BaseModel):
lines.append(f"- measured silhouette IoU {self.iou:.3f}"
+ (f", aspect error {self.aspect_ratio_err:.0%}" if self.aspect_ratio_err is not None else "")
+ ("" if self.reliable else " (mask unreliable — judge visually)"))
- for m in sorted(self.mismatches, key=lambda x: _SEVERITY_ORDER.get(x.severity, 3))[:n]:
+ for m in self.top(n):
lines.append(f"- {m.as_line()}")
if self.matches:
lines.append("- already matching: " + "; ".join(self.matches[:4]))
diff --git a/harness/codeverse/skills/__init__.py b/harness/codeverse/skills/__init__.py
index e534d1f6..04e38aec 100644
--- a/harness/codeverse/skills/__init__.py
+++ b/harness/codeverse/skills/__init__.py
@@ -107,11 +107,10 @@ def load_skill(name: str, root: Path | None = None) -> Skill:
from codeverse.skills.config import skills_enabled, skills_max, skills_unverified # noqa: E402
from codeverse.skills.materialize import attach_skills, materialize_skills # noqa: E402
from codeverse.skills.registry import plan_signals, select, skills_for # noqa: E402
-from codeverse.skills.targets import check_claims, claim_values # noqa: E402
+from codeverse.skills.targets import check_claims # noqa: E402
from codeverse.skills.telemetry import probe_reads # noqa: E402
__all__ = ["SKILLS_DIR_ENV", "Skill", "SkillError", "SkillRef", "all_skills", "attach_skills", "bundle_dirs",
- "check_claims", "claim_values",
- "iter_skills", "load_skill", "materialize_skills", "plan_signals", "probe_reads", "select",
+ "check_claims", "iter_skills", "load_skill", "materialize_skills", "plan_signals", "probe_reads", "select",
"skills_dir", "skills_enabled", "skills_for", "skills_max", "skills_unverified",
"validate_bundle"]
diff --git a/harness/codeverse/skills/targets.py b/harness/codeverse/skills/targets.py
index a13d1f93..5fc519e7 100644
--- a/harness/codeverse/skills/targets.py
+++ b/harness/codeverse/skills/targets.py
@@ -68,14 +68,6 @@ class Target:
measurable: bool = True
caveat: str = ""
- @property
- def frontmatter(self) -> dict[str, str]:
- """The rows a bundle must carry under ``metadata`` (spec-legal string values)."""
- return {"target_metric": self.metric,
- "target_direction": self.direction,
- "target_unit": self.unit,
- "target_measurable": "true" if self.measurable else "false"}
-
OBJECT_LANGS = ("blender", "urdf_blender", "cadquery", "threejs")
@@ -376,11 +368,6 @@ def check_claims(skill: Skill, root: Path | None = None) -> list[str]:
return issues
-def claim_values(name: str, root: Path | None = None) -> dict[str, str]:
- """``key -> text`` for one bundle — the rendered string, in that bundle's own units."""
- return {str(r["key"]): str(r.get("text", "")) for r in load_claims(name, root) if r.get("key")}
-
-
def claim_bases(name: str, root: Path | None = None) -> dict[str, tuple[str, Any]]:
"""``key -> (dotted target, live value)`` — the agreement test's real comparison.
diff --git a/harness/docs/INTERFACES.md b/harness/docs/INTERFACES.md
index 341bc0f8..7d080a9e 100644
--- a/harness/docs/INTERFACES.md
+++ b/harness/docs/INTERFACES.md
@@ -537,8 +537,8 @@ from codeverse.gallery.index import hero_view # (ws, rec) -> (rel, labe
from codeverse.flywheel.record import complexity_block, round_complexity # objective complexity of what shipped
# finalize_record fills record.extra["complexity"] = the BEST round's vector + plan_parts /
# parts_per_plan_part / by_round; every rounds_summary row gains "complexity" (the index or None)
-from codeverse.flywheel.export import export_samples # (runs_dir, out_dir, *, min_score=None, only_passed=False, best_round=True,
- # overwrite=True, include_unbuilt=False, captions_dir=None, drop_duplicates=False) -> ExportReport{…, n_duplicates, duplicates, tiers}
+from codeverse.flywheel.export import export_samples # (runs_dir, out_dir, *, min_score=None, only_passed=False,
+ # include_unbuilt=False, captions_dir=None, drop_duplicates=False) -> ExportReport{…, n_duplicates, duplicates, tiers}
from codeverse.flywheel.quality import quality_tier, prompt_hash, find_duplicates # tiers: A passed & 0 gate errors, B passed,
# C best ≥ 0.6, D else; dedupe = (code fingerprint, prompt)
from codeverse.flywheel.pairs import build_pairs # (runs_dir, out_jsonl, *, min_delta=0.05) -> n
diff --git a/harness/tests/flywheel_cli/test_manifest_pack.py b/harness/tests/flywheel_cli/test_manifest_pack.py
index 44d82433..4212d983 100644
--- a/harness/tests/flywheel_cli/test_manifest_pack.py
+++ b/harness/tests/flywheel_cli/test_manifest_pack.py
@@ -23,7 +23,6 @@ import pytest
from codeverse.flywheel.export import (
MANIFEST_NAME,
ManifestError,
- collect_rows,
export_samples,
load_manifest,
write_manifest,
@@ -175,8 +174,6 @@ def test_a_tampered_sample_file_fails_fast_and_names_it(exported_ds: Path):
def test_planted_src_meta_json_is_not_a_row_and_not_packed(exported_ds: Path):
sdir = exported_ds / "static_object" / "blender" / "wooden_chair_ab12cd34"
(sdir / "src" / "meta.json").write_text(json.dumps({"id": "phantom", "key": "phantom"}))
- rows = collect_rows(exported_ds) # the recovery/debug rescan is exactly 3 levels deep
- assert len(rows) == 3 and all(r["id"] != "phantom" for r in rows)
pack_samples(exported_ds) # and pack never looks at the directory tree at all
assert verify_locators(exported_ds) == 3
assert "wooden_chair_ab12cd34/src/meta.json" not in _tar_member_names(exported_ds)
diff --git a/harness/tests/flywheel_cli/test_record_env.py b/harness/tests/flywheel_cli/test_record_env.py
index 691d1fc8..e9d57bfb 100644
--- a/harness/tests/flywheel_cli/test_record_env.py
+++ b/harness/tests/flywheel_cli/test_record_env.py
@@ -38,6 +38,9 @@ def test_harness_git_sha_from_a_tracked_file_and_dirty_marker(tmp_path):
pkg.mkdir(parents=True)
mod = pkg / "record.py"
mod.write_text("# tracked\n")
+ renderer = repo / "harness" / "runtime_js" / "render.cjs"
+ renderer.parent.mkdir()
+ renderer.write_text("// tracked\n")
_git("init", "-q", cwd=repo)
_git("-c", "user.email=t@t", "-c", "user.name=t", "add", "-A", cwd=repo)
_git("-c", "user.email=t@t", "-c", "user.name=t", "commit", "-q", "-m", "init", cwd=repo)
@@ -48,6 +51,11 @@ def test_harness_git_sha_from_a_tracked_file_and_dirty_marker(tmp_path):
assert _harness_git_sha(mod) == head
mod.write_text("# edited\n")
assert _harness_git_sha(mod) == head + "-dirty"
+ # harness/ is the tree, not harness/codeverse: an edited renderer changes what a run measures
+ _git("checkout", "-q", "--", ".", cwd=repo)
+ assert _harness_git_sha(mod) == head
+ renderer.write_text("// edited\n")
+ assert _harness_git_sha(mod) == head + "-dirty"
def test_harness_git_sha_is_empty_for_an_untracked_copy(tmp_path):
diff --git a/harness/tests/flywheel_cli/test_run_identity.py b/harness/tests/flywheel_cli/test_run_identity.py
index c5524da8..0a08b8f8 100644
--- a/harness/tests/flywheel_cli/test_run_identity.py
+++ b/harness/tests/flywheel_cli/test_run_identity.py
@@ -19,7 +19,7 @@ from codeverse.contracts.run import RunId
from codeverse.flywheel.export import export_samples, load_captions
from codeverse.flywheel.index import build_index as build_sqlite_index
from codeverse.flywheel.pairs import build_pairs
-from codeverse.flywheel.record import FoundRun, battery_label, iter_runs, run_id_for
+from codeverse.flywheel.record import FoundRun, battery_label, iter_runs, run_id_for, unique_files
from codeverse.flywheel.sample import SampleError
from tests.flywheel_cli.conftest import make_fake_run
@@ -65,6 +65,15 @@ def test_battery_label_and_run_id_for(tmp_path: Path):
assert rid.battery == "static_v9" and rid.rel == "stool_a1" and rid.slug == "stool_a1"
+def test_unique_files_skips_subruns_below_the_root_not_above_it(tmp_path: Path):
+ """A scan rooted INSIDE ``_assets`` used to return [] — the skip read absolute parts."""
+ root = tmp_path / "scene" / "_assets" / "lamp"
+ (root / "_cand" / "c1").mkdir(parents=True)
+ (root / "record.json").write_text("{}")
+ (root / "_cand" / "c1" / "record.json").write_text("{}")
+ assert unique_files(root, "record.json") == [root / "record.json"]
+
+
# --------------------------------------------------------------------------- shared tree
def _nested_battery(tmp_path: Path) -> Path:
"""ONE root, two real runs whose directories are BOTH named ``run``."""
diff --git a/harness/tests/gallery/test_index.py b/harness/tests/gallery/test_index.py
index e7abfc84..8175b10c 100644
--- a/harness/tests/gallery/test_index.py
+++ b/harness/tests/gallery/test_index.py
@@ -5,7 +5,8 @@ from __future__ import annotations
import json
from pathlib import Path
-from codeverse.gallery.index import build_index, default_roots, entry_for_dir, root_label
+from codeverse.flywheel.record import battery_label
+from codeverse.gallery.index import build_index, default_roots, entry_for_dir
from codeverse.gallery.model import match, sort_entries, summarize
@@ -82,7 +83,7 @@ def test_filter_and_sort_and_summary(gallery_tree: dict[str, Path]):
def test_default_roots_and_labels(gallery_tree: dict[str, Path]):
base = gallery_tree["root"]
roots = default_roots(base)
- assert [root_label(r) for r in roots] == ["runs", "static_v9"]
+ assert [battery_label(r) for r in roots] == ["runs", "static_v9"]
assert default_roots(base / "nope") == []
diff --git a/harness/tests/gallery/test_triage.py b/harness/tests/gallery/test_triage.py
index 56a4b595..49bcf08b 100644
--- a/harness/tests/gallery/test_triage.py
+++ b/harness/tests/gallery/test_triage.py
@@ -10,6 +10,7 @@ from __future__ import annotations
import json
from pathlib import Path
+from urllib.parse import parse_qs
import pytest
@@ -33,7 +34,7 @@ def test_the_four_buckets_are_disjoint_and_sum_to_n(gallery_tree: dict[str, Path
s = summarize(entries)
assert s.n == 6
assert set(s.breakdown) == set(VERDICTS)
- assert sum(s.breakdown.values()) == s.n and s.sums()
+ assert sum(s.breakdown.values()) == s.n
# the tree holds 3 passing, 1 failing, and 2 runs with no usable record
assert s.breakdown == {"passed": 3, "failed": 1, "unjudged": 0, "error": 2}
# every entry lands in exactly one bucket
@@ -52,7 +53,7 @@ def test_a_run_with_no_record_is_error_not_unjudged(gallery_tree: dict[str, Path
def test_breakdown_of_an_empty_selection_is_all_zeroes():
b = verdict_breakdown([])
assert b == dict.fromkeys(VERDICTS, 0) and sum(b.values()) == 0
- assert summarize([]).sums()
+ assert summarize([]).n == 0
def test_the_page_states_the_breakdown_and_it_adds_up(gallery_tree: dict[str, Path]):
@@ -168,6 +169,17 @@ def test_compare_reports_unknown_keys_and_caps_the_width(gallery_tree: dict[str,
== min(MAX_COMPARE, len(entries))
+def test_query_links_survive_a_duplicate_root_label(gallery_tree: dict[str, Path]):
+ """``build_index`` mints ``runs#2``; an unquoted ``#`` ends the URL in the browser."""
+ app = GalleryApp([gallery_tree["runs"], gallery_tree["runs"]])
+ key = "runs#2/wooden_chair_ab12cd34"
+ compare = app.route("/compare", {"runs": key}).body.decode()
+ href = compare.split("href='/export.csv?", 1)[1].split("'", 1)[0]
+ assert "#" not in href and parse_qs(href) == {"runs": [key]}
+ detail = app.route("/run/runs#2/wooden_chair_ab12cd34").body.decode()
+ assert "href='/?battery=runs%232'" in detail
+
+
def test_compare_survives_a_run_with_no_record(gallery_tree: dict[str, Path]):
app = GalleryApp([gallery_tree["battery"]])
r = app.route("/compare", {"runs": "static_v9/half_written,static_v9/not_started"})