Official Codebase of 3DCodeVerse
Python
skills: freshness + value-level contradiction tests, and the three contract.md overlap claims they caught
gaoypeng6b477db46fd5
diff --git a/harness/codeverse/prompts/blender/contract.md b/harness/codeverse/prompts/blender/contract.mdindex 96a95189..a5d6cd9e 100644--- a/harness/codeverse/prompts/blender/contract.md+++ b/harness/codeverse/prompts/blender/contract.md@@ -26,7 +26,7 @@plan says. Auto-suffixed duplicates (`Leg.001`) violate it: name instances `<Name>_0 … <Name>_N-1`.* Meshes only (no empties as parts, no un-converted curves, no cameras/lights); link everyobject to `bpy.context.scene.collection`. Plan numbers = constants at the top of each part- file; keep each part inside its plan bbox; attached parts touch (2–4 mm overlap is fine).+ file; keep each part inside its plan bbox; attached parts overlap 0.5–2 mm (deeper = interpenetration).## Allowed imports · forbidden calls · limits* Imports: `bpy`, `bmesh`, `mathutils`, `math`, `random`, `itertools`, `functools`, `typing`,@@ -65,7 +65,7 @@ def build_seat() -> bpy.types.Object:import bpy, bmesh, mathLEG_R, LEG_N, LEG_RING = 0.02, 3, 0.12 # metres, from the plan-LEG_H = 0.45 - 0.04 + 0.004 # reaches 4 mm into the seat (weld)+LEG_H = 0.45 - 0.04 + 0.001 # reaches 1 mm into the seat (weld)def build_leg() -> list[bpy.types.Object]:"""Leg x3 — steel rods Leg_0..Leg_2 on a ring, each TOP-LEVEL (no parent Empty!)."""@@ -131,7 +131,7 @@ def build_seat() -> bpy.types.Object:return seatdef build_leg(i: int) -> bpy.types.Object:- a, h = 2 * math.pi * i / LEG_N, SEAT_Z - SEAT_T + 0.004 # 4 mm into the seat (weld)+ a, h = 2 * math.pi * i / LEG_N, SEAT_Z - SEAT_T + 0.001 # 1 mm into the seat (weld)leg = make_cylinder(f"Leg{i + 1}", LEG_R, h, (LEG_RING * math.cos(a), LEG_RING * math.sin(a), h / 2), 24)leg.data.materials.append(make_material(f"Steel{i + 1}", (0.6, 0.6, 0.62), 0.35, 1.0))return legdiff --git a/harness/codeverse/prompts/cadquery/contract.md b/harness/codeverse/prompts/cadquery/contract.mdindex 4569a0d8..7f105d7c 100644--- a/harness/codeverse/prompts/cadquery/contract.md+++ b/harness/codeverse/prompts/cadquery/contract.md@@ -31,7 +31,8 @@ No `OCP`/`OCC` direct calls, no numpy, no file I/O, no `cq.exporters`, no* ≤ 40 parts; each part a valid solid (`.val().isValid()`), no empty Workplanes; fillets< 0.45 × thinnest adjacent wall; feature sizes ≥ 0.5 mm; build < 120 s.* Do not union parts across plan boundaries (keep one solid per part); small intentional- overlap (≥ 2 mm) between touching parts is required — that is how they "weld".+ overlap of **0.5–2 mm** between touching parts is required — that is how they "weld";+ deeper than 2 mm and the connectivity gate calls it interpenetration.## Self-check (put at the end of the file)```pythondiff --git a/harness/codeverse/prompts/threejs/contract.md b/harness/codeverse/prompts/threejs/contract.mdindex 4e95e90b..7ab60c29 100644--- a/harness/codeverse/prompts/threejs/contract.md+++ b/harness/codeverse/prompts/threejs/contract.md@@ -80,7 +80,7 @@ export function buildSeat(THREE) {`src/parts/legs.js````jsimport * as THREE from 'three';-const LEG_R = 0.018, LEG_H = 0.45 - 0.04 + 0.004; // reaches 4 mm into the seat (weld)+const LEG_R = 0.018, LEG_H = 0.45 - 0.04 + 0.001; // reaches 1 mm into the seat (weld)const LEG_XZ = 0.15;export function buildLegs(THREE) {const g = new THREE.Group(); g.name = 'Legs';diff --git a/harness/codeverse/skills/claims.py b/harness/codeverse/skills/claims.pyindex f7b057a3..e002787e 100644--- a/harness/codeverse/skills/claims.py+++ b/harness/codeverse/skills/claims.py@@ -97,8 +97,27 @@ def check_claims(skill: Skill, root: Path | None = None) -> list[str]:def claim_values(name: str, root: Path | None = None) -> dict[str, str]:- """``key -> text`` for one bundle — what the agreement test compares across skills."""+ """``key -> text`` for one bundle — the rendered string, in that bundle's own units."""return {str(r["key"]): str(r.get("text", "")) for r in load_claims(name, root) if r.get("key")}-__all__ = ["CLAIMS_DIR", "check_claims", "claim_values", "claims_path", "load_claims", "render_claim", "resolve"]+def claim_bases(name: str, root: Path | None = None) -> dict[str, tuple[str, Any]]:+ """``key -> (dotted target, live value)`` — the agreement test's real comparison.++ WHY not the rendered text: two skills may honestly quote one constant in two units.+ ``cv3d-bbox-contract`` says "1 cm" (scale 100) and ``cv3d-repeats-and-mirrors`` says+ "0.01" metres; both pin ``conventions:BBOX_TOLERANCE_M`` and both are right. A+ contradiction is two skills pointing a shared key at DIFFERENT numbers, so that is what+ is compared — the pre-scale value, and the target it came from.+ """+ out: dict[str, tuple[str, Any]] = {}+ for row in load_claims(name, root):+ key = row.get("key")+ if not key or not row.get("python"):+ continue+ out[str(key)] = (str(row["python"]), resolve(str(row["python"])))+ return out+++__all__ = ["CLAIMS_DIR", "check_claims", "claim_bases", "claim_values", "claims_path", "load_claims",+ "render_claim", "resolve"]diff --git a/harness/codeverse/skills/prompting.py b/harness/codeverse/skills/prompting.pyindex 95c89599..7a549dde 100644--- a/harness/codeverse/skills/prompting.py+++ b/harness/codeverse/skills/prompting.py@@ -22,6 +22,7 @@ a score claim in every prompt, and a score claim belongs in a measured report.from __future__ import annotations+import refrom collections.abc import Sequencefrom codeverse.cost.guard import text_tokens@@ -40,6 +41,24 @@ MANDATE = ("Skills for this task are installed in this workspace. Any whose desc_HEADING = "## Skills"+#: how much of a description the api-agent index repeats. The spec lets a description run to+#: 1024 chars and ours use it — they are what a CLI's own matcher reads. api-agent's index is+#: not a matcher: OUR router already decided, so the line only has to be recognisable enough+#: for the agent to know which file to open. Measured on the real library, quoting the full+#: 14 descriptions costs 780 tokens for five skills; the first clause costs 401.+INDEX_SUMMARY_CHARS = 200++_SENTENCE_END = re.compile(r"(?<=[.!?])\s+(?=[A-Z`])")+++def index_summary(description: str, limit: int = INDEX_SUMMARY_CHARS) -> str:+ """The WHAT half of a description, for the one index we write ourselves."""+ first = _SENTENCE_END.split(description.strip(), 1)[0].strip()+ if len(first) <= limit:+ return first+ cut = first[:limit].rsplit(" ", 1)[0].rstrip(" ,;:—-")+ return cut + " ..."+def skill_path(name: str, *, agent_kind: str = "") -> str:root = CLAUDE_SKILL_ROOT if agent_kind == "claude-code" else AGENTS_SKILL_ROOT@@ -57,7 +76,8 @@ def index_block(skills: Sequence[Skill | Selection], agent_kind: str) -> str:return ""if agent_kind in NATIVE_LOADERS:return f"{_HEADING}\n\n{MANDATE}\n"- lines = [f"- **{s.name}** — {s.description} Read `{skill_path(s.name, agent_kind=agent_kind)}` BEFORE writing code."+ lines = [f"- **{s.name}** — {index_summary(s.description)} "+ f"Read `{skill_path(s.name, agent_kind=agent_kind)}` BEFORE writing code."for s in items]return f"{_HEADING}\n\n{MANDATE}\n\n" + "\n".join(lines) + "\n"@@ -99,5 +119,6 @@ def inline_body(selections: Sequence[Selection], *, max_tokens: int = 2500) -> treturn "", ""-__all__ = ["AGENTS_SKILL_ROOT", "CLAUDE_SKILL_ROOT", "MANDATE", "NATIVE_LOADERS",- "index_block", "index_tokens", "inline_body", "repair_pointers", "skill_path"]+__all__ = ["AGENTS_SKILL_ROOT", "CLAUDE_SKILL_ROOT", "INDEX_SUMMARY_CHARS", "MANDATE",+ "NATIVE_LOADERS", "index_block", "index_summary", "index_tokens", "inline_body",+ "repair_pointers", "skill_path"]diff --git a/harness/tests/skills/test_freshness.py b/harness/tests/skills/test_freshness.pynew file mode 100644index 00000000..16b37d4f--- /dev/null+++ b/harness/tests/skills/test_freshness.py@@ -0,0 +1,228 @@+"""Freshness: every live name a bundle quotes must still exist in the code today.++WHY this file is the reason the library can be trusted in a month. A skill's whole+value is that its names and numbers are OUR names and numbers; a sentence naming a tool+we renamed, a gate kind we retired or a constant we deleted is not merely useless, it+sends the agent to look for something that is not there and costs a turn to find out.+``test_library`` pins the numbers a bundle DECLARES in ``_claims``; this file pins the+vocabulary it merely mentions, which is the part an author can forget to declare.++The four vocabularies are checked, not guessed at: harness tool names live in the tool+registry, gate kinds in ``skills.registry``, cookbook sections in the language cookbook+the ``read_cookbook`` tool actually serves, and constants in the source. Anything from a+foreign namespace (bpy, GL, GLSL, three.js) is listed in :data:`FOREIGN` with the library+it belongs to — an allowlist an author has to extend deliberately, so the test keeps+meaning something.+"""++from __future__ import annotations++import re+from pathlib import Path++import pytest++from codeverse.skills import bundle_dirs, iter_skills, skills_dir+from codeverse.skills.registry import ROUTED_SKILLS, ROUTES+from codeverse.spatial.registry import list_tools++BUNDLES = bundle_dirs()+SKILLS = list(iter_skills()) if BUNDLES else []+pytestmark = pytest.mark.skipif(not BUNDLES, reason=f"no bundles in {skills_dir()} yet")++HARNESS = Path(__file__).resolve().parents[2]++#: every markdown file a bundle ships — SKILL.md and its references are equally quotable+DOCS: list[Path] = sorted(p for d in BUNDLES for p in d.rglob("*.md"))++_FENCE = re.compile(r"^```.*?^```", re.S | re.M)+_INLINE_CODE = re.compile(r"`([^`\n]+)`")+_CONST = re.compile(r"\b[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)+\b")+_GATE_SLUG = re.compile(+ r"\b(?:connectivity|contract|joint_sweep|motion_direction|scene_frames|gl_frames|lint"+ r"|shader|detail_drift|render_console|shader_preflight)/[a-z_]+\b")+#: a snake_case token in backticks is CLAIMED to be a name in this harness. Which live+#: vocabulary it belongs to does not matter — that it belongs to one of them does.+_HARNESS_SHAPED = re.compile(r"^[a-z][a-z0-9]*(?:_[a-z0-9]+)+$")+_COOKBOOK_SECTION = re.compile(r'read_cookbook\(\s*section\s*=\s*"([^"]+)"')+_ENV = re.compile(r"\bCV3D_[A-Z0-9_]+\b")++#: ALL_CAPS tokens that are NOT ours, with the namespace that owns them. Extend+#: deliberately: an author who adds a name here is asserting it is somebody else's.+FOREIGN: dict[str, str] = {+ "SIMPLE_DEFORM": "bpy modifier type enum",+ "DEPTH_TEST": "OpenGL capability (GL_DEPTH_TEST)",+ "GL_QUADS": "legacy OpenGL primitive enum",+ "USE_LOGDEPTHBUF": "three.js shader define",+ "USE_FOG": "three.js shader define",+ # variables inside a bundle's own worked examples+ "SEAT_T": "example constant", "SEAT_Z": "example constant",+ "X_0": "example part name", "X_3": "example part name",+ "CV3D_SKILLS_UNVERIFIED": "our own switch, named in prose as an env var",+}++LIVE_TOOLS = {t.name for t in list_tools()}+++++def _module_constants() -> set[str]:+ out: set[str] = set()+ for p in (HARNESS / "codeverse").rglob("*.py"):+ for m in re.finditer(r"^([A-Z][A-Z0-9_]*)\s*[:=]", p.read_text(errors="ignore"), re.M):+ out.add(m.group(1))+ return out+++LIVE_CONSTANTS = _module_constants()+++def _live_gate_kinds() -> set[str]:+ from codeverse.skills import registry++ return {v for k, v in vars(registry).items()+ if k.isupper() and isinstance(v, str) and "/" in v}+++LIVE_KINDS = _live_gate_kinds()+++def _live_vocabulary() -> set[str]:+ """Every word that appears anywhere in this harness's own source and prompts.++ Deliberately the WHOLE corpus rather than a curated list of tools: a skill quotes+ tool names, gate ids, rubric criteria, ``fix_hint`` field names, uniform names the+ glsl contract defines, lint rule ids and battery names, and every one of those is a+ checkable claim about something that exists. Curating them by hand would mean the+ author of the next vocabulary has to remember to extend the test; asking "does this+ word occur anywhere in what we ship" needs no maintenance and still fails loudly the+ day a name is renamed out of the codebase.+ """+ words: set[str] = set()+ for root, exts in ((HARNESS / "codeverse", {".py", ".md", ".j2", ".yaml", ".yml", ".toml"}),+ (HARNESS / "bench", {".py", ".yaml", ".yml"}),+ (HARNESS / "runtime_js", {".js", ".mjs", ".glsl", ".json"})):+ if not root.is_dir():+ continue+ for p in root.rglob("*"):+ if (not p.is_file() or p.suffix not in exts+ or "node_modules" in p.parts or "__pycache__" in p.parts+ or p.parent.name.startswith("cv3d-") or "_claims" in p.parts+ or p.parent.parent.name.startswith("cv3d-")):+ continue+ words |= set(re.findall(r"[A-Za-z_][A-Za-z0-9_]*", p.read_text(errors="ignore")))+ out = HARNESS / "bench" / "out" # battery names a bundle cites as its evidence+ if out.is_dir():+ words |= {d.name for d in out.iterdir() if d.is_dir()}+ return words+++LIVE_VOCABULARY = _live_vocabulary()++#: snake_case names that are NOT ours, with who owns them. Extend deliberately.+FOREIGN_CALLS = {+ # GLSL / three.js / bpy builtins a bundle quotes while teaching that language+ "texture", "smoothstep", "fract", "mix", "clamp", "dot", "cross", "normalize", "length",+ "abs", "max", "min", "mod", "pow", "sin", "cos", "step", "reflect", "refract",+ "transform_apply", "primitive_cube_add", "ensure_lookup_table", "hide_set", "update",+ "len", "setup", "range", "print", "float", "int",+ # illustrative names inside a bundle's own worked example+ "left_wheel", "right_wheel",+ # batteries whose results are cited as evidence but whose output dir is gitignored+ "graphics_v1_flash", "graphics_v2_flash", "scenes_v1_flash",+}+++def _squash(text: str) -> str:+ """One-line, lowercase — a section name wrapped across two markdown lines is one name."""+ return " ".join(text.lower().split())+++def _prose(text: str) -> str:+ """The body with fenced code removed — a define inside a shader is not our vocabulary."""+ return _FENCE.sub("", text)++[email protected]("doc", DOCS, ids=[str(p.relative_to(skills_dir())) for p in DOCS])+def test_every_harness_name_a_bundle_quotes_is_still_a_live_name(doc: Path):+ """`check_connectivity` must not become a name only the skill remembers.++ Wider than the tool registry on purpose: a rubric criterion a skill tells the agent to+ optimise is as much a live name as a tool, and rubric criteria get renamed.+ """+ text = _prose(doc.read_text())+ named = {t.strip("`").split("(")[0].strip() for t in _INLINE_CODE.findall(text)}+ named = {t for t in named if _HARNESS_SHAPED.match(t)}+ unknown = sorted(named - LIVE_VOCABULARY - FOREIGN_CALLS)+ assert unknown == [], (+ f"{doc.name} quotes {unknown}, which no tool, gate, rubric, criterion, language or "+ f"contract entry point answers to today")++[email protected]("doc", DOCS, ids=[str(p.relative_to(skills_dir())) for p in DOCS])+def test_every_gate_kind_a_bundle_names_is_a_live_kind(doc: Path):+ """A reworded gate that changes a slug must break here, not silently unroute a sheet."""+ slugs = set(_GATE_SLUG.findall(_prose(doc.read_text())))+ unknown = sorted(slugs - LIVE_KINDS)+ assert unknown == [], f"{doc.name} names gate kind(s) registry.py does not define: {unknown}"++[email protected]("doc", DOCS, ids=[str(p.relative_to(skills_dir())) for p in DOCS])+def test_every_constant_a_bundle_names_in_prose_is_ours_or_declared_foreign(doc: Path):+ toks = set(_CONST.findall(_prose(doc.read_text())))+ unknown = sorted(t for t in toks if t not in LIVE_CONSTANTS and t not in FOREIGN)+ assert unknown == [], (+ f"{doc.name} names {unknown}: either the constant moved, or add it to FOREIGN "+ f"with the namespace that owns it")++[email protected]("doc", DOCS, ids=[str(p.relative_to(skills_dir())) for p in DOCS])+def test_every_switch_a_bundle_names_is_a_live_switch(doc: Path):+ from codeverse.tracks.plan_features import LIVE_SWITCHES++ live = {s if isinstance(s, str) else getattr(s, "env", "") for s in LIVE_SWITCHES}+ live |= {n for n in LIVE_CONSTANTS if n.startswith("CV3D_")}+ live |= {"CV3D_SKILLS", "CV3D_SKILLS_MAX", "CV3D_SKILLS_UNVERIFIED", "CV3D_SKILLS_DIR",+ "CV3D_MAX_IN_FLIGHT"}+ named = set(_ENV.findall(doc.read_text()))+ assert named <= live, f"{doc.name} names dead switch(es): {sorted(named - live)}"++[email protected]("doc", DOCS, ids=[str(p.relative_to(skills_dir())) for p in DOCS])+def test_every_sibling_skill_a_bundle_names_exists(doc: Path):+ named = set(re.findall(r"\bcv3d-[a-z0-9-]+\b", doc.read_text()))+ have = {d.name for d in BUNDLES} | set(ROUTED_SKILLS)+ assert named <= have, f"{doc.name} points at missing skill(s): {sorted(named - have)}"+++def _languages_of(skill_name: str) -> set[str]:+ out: set[str] = set()+ for r in ROUTES:+ if r.skill == skill_name:+ out |= set(r.languages)+ return out++[email protected]("s", SKILLS, ids=[s.name for s in SKILLS])+def test_every_cookbook_section_a_skill_names_exists_in_that_cookbook(s):+ """T4's other half: a skill may point at the cookbook, never invent a heading."""+ from codeverse.prompts import load_text+ from codeverse.spatial.cookbook_tool import _PROMPT_DIR, split_sections++ text = "\n".join([s.body] + [p.read_text() for p in sorted(s.dir.rglob("references/*.md"))])+ wanted = set(_COOKBOOK_SECTION.findall(text))+ if not wanted:+ pytest.skip("names no cookbook section")+ langs = _languages_of(s.name) or {"blender"}+ titles: set[str] = set()+ for lang in langs:+ d = _PROMPT_DIR.get(lang, lang)+ try:+ md = load_text(f"{d}/cookbook.md")+ except (FileNotFoundError, OSError):+ continue+ titles |= {sec.title.lower() for sec in split_sections(md)}+ for want in sorted(_squash(w) for w in wanted):+ assert any(want in _squash(t) for t in titles), (+ f"{s.name} sends the agent to read_cookbook(section={want!r}), "+ f"which no cookbook of {sorted(langs)} has")diff --git a/harness/tests/skills/test_library.py b/harness/tests/skills/test_library.pyindex ef7788fd..44a5b8b2 100644--- a/harness/tests/skills/test_library.py+++ b/harness/tests/skills/test_library.py@@ -18,7 +18,7 @@ from pathlib import Pathimport pytestfrom codeverse.skills import bundle_dirs, iter_skills, skills_dir, validate_bundle-from codeverse.skills.claims import check_claims, claim_values+from codeverse.skills.claims import check_claims, claim_bases, load_claimsfrom codeverse.skills.loader import BODY_MAX_LINES, BODY_MAX_TOKENSfrom codeverse.skills.registry import ROUTED_SKILLS, ROUTES@@ -73,18 +73,37 @@ def test_an_evidence_thin_bundle_says_so(s):assert s.metadata.get("evidence_note"), f"{s.name}: a non-measured bundle must say why in metadata"-def test_co_routing_skills_never_disagree_about_a_number():- """T3: astra3d checked one hand-picked pair; this checks the whole co-routing graph."""- by_skill = {s.name: claim_values(s.name) for s in SKILLS}- pairs: set[tuple[str, str]] = set()- for a in by_skill:- for b in by_skill:- if a < b and _can_co_route(a, b):- pairs.add((a, b))- for a, b in sorted(pairs):- shared = set(by_skill[a]) & set(by_skill[b])- for key in sorted(shared):- assert by_skill[a][key] == by_skill[b][key], f"{a} and {b} disagree about {key}"+def test_no_two_skills_anywhere_point_one_claim_key_at_different_numbers():+ """T3: astra3d checked one hand-picked pair by hand; this checks the whole library.++ Compared on the PRE-SCALE value, not the rendered text: "1 cm" and "0.01" metres are+ the same tolerance honestly quoted in two units, and failing that pair would have+ taught the next author to delete the claim rather than fix a contradiction.+ """+ by_key: dict[str, dict[str, tuple[str, object]]] = defaultdict(dict)+ for s in SKILLS:+ for key, base in claim_bases(s.name).items():+ by_key[key][s.name] = base+ for key, owners in sorted(by_key.items()):+ targets = {t for t, _ in owners.values()}+ values = {repr(v) for _, v in owners.values()}+ assert len(targets) == 1, f"{key} is pinned to {sorted(targets)} by {sorted(owners)}"+ assert len(values) == 1, f"{key} resolves to {sorted(values)} across {sorted(owners)}"+++def test_co_routing_skills_render_a_shared_number_the_same_way():+ """A weaker but still useful rule for skills that can land in ONE session together:+ if they chose the same units for a shared key, the sentence must read the same."""+ rows = {s.name: {r["key"]: r for r in load_claims(s.name) if r.get("key")} for s in SKILLS}+ for a in sorted(rows):+ for b in sorted(rows):+ if a >= b or not _can_co_route(a, b):+ continue+ for key in sorted(set(rows[a]) & set(rows[b])):+ ra, rb = rows[a][key], rows[b][key]+ if (ra.get("scale"), ra.get("format")) != (rb.get("scale"), rb.get("format")):+ continue+ assert ra.get("text") == rb.get("text"), f"{a} and {b} disagree about {key}"def _can_co_route(a: str, b: str) -> bool: