Skip to content

Official Codebase of 3DCodeVerse

Python

skills: freshness + value-level contradiction tests, and the three contract.md overlap claims they caught

gaoypeng6b477db46fd5
diff --git a/harness/codeverse/prompts/blender/contract.md b/harness/codeverse/prompts/blender/contract.md
index 96a95189..a5d6cd9e 100644
--- a/harness/codeverse/prompts/blender/contract.md
+++ b/harness/codeverse/prompts/blender/contract.md
@@ -26,7 +26,7 @@
plan says. Auto-suffixed duplicates (`Leg.001`) violate it: name instances `<Name>_0 … <Name>_N-1`.
* Meshes only (no empties as parts, no un-converted curves, no cameras/lights); link every
object to `bpy.context.scene.collection`. Plan numbers = constants at the top of each part
- file; keep each part inside its plan bbox; attached parts touch (2–4 mm overlap is fine).
+ file; keep each part inside its plan bbox; attached parts overlap 0.5–2 mm (deeper = interpenetration).
## Allowed imports · forbidden calls · limits
* Imports: `bpy`, `bmesh`, `mathutils`, `math`, `random`, `itertools`, `functools`, `typing`,
@@ -65,7 +65,7 @@ def build_seat() -> bpy.types.Object:
import bpy, bmesh, math
LEG_R, LEG_N, LEG_RING = 0.02, 3, 0.12 # metres, from the plan
-LEG_H = 0.45 - 0.04 + 0.004 # reaches 4 mm into the seat (weld)
+LEG_H = 0.45 - 0.04 + 0.001 # reaches 1 mm into the seat (weld)
def build_leg() -> list[bpy.types.Object]:
"""Leg x3 — steel rods Leg_0..Leg_2 on a ring, each TOP-LEVEL (no parent Empty!)."""
@@ -131,7 +131,7 @@ def build_seat() -> bpy.types.Object:
return seat
def build_leg(i: int) -> bpy.types.Object:
- a, h = 2 * math.pi * i / LEG_N, SEAT_Z - SEAT_T + 0.004 # 4 mm into the seat (weld)
+ a, h = 2 * math.pi * i / LEG_N, SEAT_Z - SEAT_T + 0.001 # 1 mm into the seat (weld)
leg = make_cylinder(f"Leg{i + 1}", LEG_R, h, (LEG_RING * math.cos(a), LEG_RING * math.sin(a), h / 2), 24)
leg.data.materials.append(make_material(f"Steel{i + 1}", (0.6, 0.6, 0.62), 0.35, 1.0))
return leg
diff --git a/harness/codeverse/prompts/cadquery/contract.md b/harness/codeverse/prompts/cadquery/contract.md
index 4569a0d8..7f105d7c 100644
--- a/harness/codeverse/prompts/cadquery/contract.md
+++ b/harness/codeverse/prompts/cadquery/contract.md
@@ -31,7 +31,8 @@ No `OCP`/`OCC` direct calls, no numpy, no file I/O, no `cq.exporters`, no
* ≤ 40 parts; each part a valid solid (`.val().isValid()`), no empty Workplanes; fillets
< 0.45 × thinnest adjacent wall; feature sizes ≥ 0.5 mm; build < 120 s.
* Do not union parts across plan boundaries (keep one solid per part); small intentional
- overlap (≥ 2 mm) between touching parts is required — that is how they "weld".
+ overlap of **0.5–2 mm** between touching parts is required — that is how they "weld";
+ deeper than 2 mm and the connectivity gate calls it interpenetration.
## Self-check (put at the end of the file)
```python
diff --git a/harness/codeverse/prompts/threejs/contract.md b/harness/codeverse/prompts/threejs/contract.md
index 4e95e90b..7ab60c29 100644
--- a/harness/codeverse/prompts/threejs/contract.md
+++ b/harness/codeverse/prompts/threejs/contract.md
@@ -80,7 +80,7 @@ export function buildSeat(THREE) {
`src/parts/legs.js`
```js
import * as THREE from 'three';
-const LEG_R = 0.018, LEG_H = 0.45 - 0.04 + 0.004; // reaches 4 mm into the seat (weld)
+const LEG_R = 0.018, LEG_H = 0.45 - 0.04 + 0.001; // reaches 1 mm into the seat (weld)
const LEG_XZ = 0.15;
export function buildLegs(THREE) {
const g = new THREE.Group(); g.name = 'Legs';
diff --git a/harness/codeverse/skills/claims.py b/harness/codeverse/skills/claims.py
index f7b057a3..e002787e 100644
--- a/harness/codeverse/skills/claims.py
+++ b/harness/codeverse/skills/claims.py
@@ -97,8 +97,27 @@ def check_claims(skill: Skill, root: Path | None = None) -> list[str]:
def claim_values(name: str, root: Path | None = None) -> dict[str, str]:
- """``key -> text`` for one bundle — what the agreement test compares across skills."""
+ """``key -> text`` for one bundle — the rendered string, in that bundle's own units."""
return {str(r["key"]): str(r.get("text", "")) for r in load_claims(name, root) if r.get("key")}
-__all__ = ["CLAIMS_DIR", "check_claims", "claim_values", "claims_path", "load_claims", "render_claim", "resolve"]
+def claim_bases(name: str, root: Path | None = None) -> dict[str, tuple[str, Any]]:
+ """``key -> (dotted target, live value)`` — the agreement test's real comparison.
+
+ WHY not the rendered text: two skills may honestly quote one constant in two units.
+ ``cv3d-bbox-contract`` says "1 cm" (scale 100) and ``cv3d-repeats-and-mirrors`` says
+ "0.01" metres; both pin ``conventions:BBOX_TOLERANCE_M`` and both are right. A
+ contradiction is two skills pointing a shared key at DIFFERENT numbers, so that is what
+ is compared — the pre-scale value, and the target it came from.
+ """
+ out: dict[str, tuple[str, Any]] = {}
+ for row in load_claims(name, root):
+ key = row.get("key")
+ if not key or not row.get("python"):
+ continue
+ out[str(key)] = (str(row["python"]), resolve(str(row["python"])))
+ return out
+
+
+__all__ = ["CLAIMS_DIR", "check_claims", "claim_bases", "claim_values", "claims_path", "load_claims",
+ "render_claim", "resolve"]
diff --git a/harness/codeverse/skills/prompting.py b/harness/codeverse/skills/prompting.py
index 95c89599..7a549dde 100644
--- a/harness/codeverse/skills/prompting.py
+++ b/harness/codeverse/skills/prompting.py
@@ -22,6 +22,7 @@ a score claim in every prompt, and a score claim belongs in a measured report.
from __future__ import annotations
+import re
from collections.abc import Sequence
from codeverse.cost.guard import text_tokens
@@ -40,6 +41,24 @@ MANDATE = ("Skills for this task are installed in this workspace. Any whose desc
_HEADING = "## Skills"
+#: how much of a description the api-agent index repeats. The spec lets a description run to
+#: 1024 chars and ours use it — they are what a CLI's own matcher reads. api-agent's index is
+#: not a matcher: OUR router already decided, so the line only has to be recognisable enough
+#: for the agent to know which file to open. Measured on the real library, quoting the full
+#: 14 descriptions costs 780 tokens for five skills; the first clause costs 401.
+INDEX_SUMMARY_CHARS = 200
+
+_SENTENCE_END = re.compile(r"(?<=[.!?])\s+(?=[A-Z`])")
+
+
+def index_summary(description: str, limit: int = INDEX_SUMMARY_CHARS) -> str:
+ """The WHAT half of a description, for the one index we write ourselves."""
+ first = _SENTENCE_END.split(description.strip(), 1)[0].strip()
+ if len(first) <= limit:
+ return first
+ cut = first[:limit].rsplit(" ", 1)[0].rstrip(" ,;:—-")
+ return cut + " ..."
+
def skill_path(name: str, *, agent_kind: str = "") -> str:
root = CLAUDE_SKILL_ROOT if agent_kind == "claude-code" else AGENTS_SKILL_ROOT
@@ -57,7 +76,8 @@ def index_block(skills: Sequence[Skill | Selection], agent_kind: str) -> str:
return ""
if agent_kind in NATIVE_LOADERS:
return f"{_HEADING}\n\n{MANDATE}\n"
- lines = [f"- **{s.name}** — {s.description} Read `{skill_path(s.name, agent_kind=agent_kind)}` BEFORE writing code."
+ lines = [f"- **{s.name}** — {index_summary(s.description)} "
+ f"Read `{skill_path(s.name, agent_kind=agent_kind)}` BEFORE writing code."
for s in items]
return f"{_HEADING}\n\n{MANDATE}\n\n" + "\n".join(lines) + "\n"
@@ -99,5 +119,6 @@ def inline_body(selections: Sequence[Selection], *, max_tokens: int = 2500) -> t
return "", ""
-__all__ = ["AGENTS_SKILL_ROOT", "CLAUDE_SKILL_ROOT", "MANDATE", "NATIVE_LOADERS",
- "index_block", "index_tokens", "inline_body", "repair_pointers", "skill_path"]
+__all__ = ["AGENTS_SKILL_ROOT", "CLAUDE_SKILL_ROOT", "INDEX_SUMMARY_CHARS", "MANDATE",
+ "NATIVE_LOADERS", "index_block", "index_summary", "index_tokens", "inline_body",
+ "repair_pointers", "skill_path"]
diff --git a/harness/tests/skills/test_freshness.py b/harness/tests/skills/test_freshness.py
new file mode 100644
index 00000000..16b37d4f
--- /dev/null
+++ b/harness/tests/skills/test_freshness.py
@@ -0,0 +1,228 @@
+"""Freshness: every live name a bundle quotes must still exist in the code today.
+
+WHY this file is the reason the library can be trusted in a month. A skill's whole
+value is that its names and numbers are OUR names and numbers; a sentence naming a tool
+we renamed, a gate kind we retired or a constant we deleted is not merely useless, it
+sends the agent to look for something that is not there and costs a turn to find out.
+``test_library`` pins the numbers a bundle DECLARES in ``_claims``; this file pins the
+vocabulary it merely mentions, which is the part an author can forget to declare.
+
+The four vocabularies are checked, not guessed at: harness tool names live in the tool
+registry, gate kinds in ``skills.registry``, cookbook sections in the language cookbook
+the ``read_cookbook`` tool actually serves, and constants in the source. Anything from a
+foreign namespace (bpy, GL, GLSL, three.js) is listed in :data:`FOREIGN` with the library
+it belongs to — an allowlist an author has to extend deliberately, so the test keeps
+meaning something.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+import pytest
+
+from codeverse.skills import bundle_dirs, iter_skills, skills_dir
+from codeverse.skills.registry import ROUTED_SKILLS, ROUTES
+from codeverse.spatial.registry import list_tools
+
+BUNDLES = bundle_dirs()
+SKILLS = list(iter_skills()) if BUNDLES else []
+pytestmark = pytest.mark.skipif(not BUNDLES, reason=f"no bundles in {skills_dir()} yet")
+
+HARNESS = Path(__file__).resolve().parents[2]
+
+#: every markdown file a bundle ships — SKILL.md and its references are equally quotable
+DOCS: list[Path] = sorted(p for d in BUNDLES for p in d.rglob("*.md"))
+
+_FENCE = re.compile(r"^```.*?^```", re.S | re.M)
+_INLINE_CODE = re.compile(r"`([^`\n]+)`")
+_CONST = re.compile(r"\b[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)+\b")
+_GATE_SLUG = re.compile(
+ r"\b(?:connectivity|contract|joint_sweep|motion_direction|scene_frames|gl_frames|lint"
+ r"|shader|detail_drift|render_console|shader_preflight)/[a-z_]+\b")
+#: a snake_case token in backticks is CLAIMED to be a name in this harness. Which live
+#: vocabulary it belongs to does not matter — that it belongs to one of them does.
+_HARNESS_SHAPED = re.compile(r"^[a-z][a-z0-9]*(?:_[a-z0-9]+)+$")
+_COOKBOOK_SECTION = re.compile(r'read_cookbook\(\s*section\s*=\s*"([^"]+)"')
+_ENV = re.compile(r"\bCV3D_[A-Z0-9_]+\b")
+
+#: ALL_CAPS tokens that are NOT ours, with the namespace that owns them. Extend
+#: deliberately: an author who adds a name here is asserting it is somebody else's.
+FOREIGN: dict[str, str] = {
+ "SIMPLE_DEFORM": "bpy modifier type enum",
+ "DEPTH_TEST": "OpenGL capability (GL_DEPTH_TEST)",
+ "GL_QUADS": "legacy OpenGL primitive enum",
+ "USE_LOGDEPTHBUF": "three.js shader define",
+ "USE_FOG": "three.js shader define",
+ # variables inside a bundle's own worked examples
+ "SEAT_T": "example constant", "SEAT_Z": "example constant",
+ "X_0": "example part name", "X_3": "example part name",
+ "CV3D_SKILLS_UNVERIFIED": "our own switch, named in prose as an env var",
+}
+
+LIVE_TOOLS = {t.name for t in list_tools()}
+
+
+
+
+def _module_constants() -> set[str]:
+ out: set[str] = set()
+ for p in (HARNESS / "codeverse").rglob("*.py"):
+ for m in re.finditer(r"^([A-Z][A-Z0-9_]*)\s*[:=]", p.read_text(errors="ignore"), re.M):
+ out.add(m.group(1))
+ return out
+
+
+LIVE_CONSTANTS = _module_constants()
+
+
+def _live_gate_kinds() -> set[str]:
+ from codeverse.skills import registry
+
+ return {v for k, v in vars(registry).items()
+ if k.isupper() and isinstance(v, str) and "/" in v}
+
+
+LIVE_KINDS = _live_gate_kinds()
+
+
+def _live_vocabulary() -> set[str]:
+ """Every word that appears anywhere in this harness's own source and prompts.
+
+ Deliberately the WHOLE corpus rather than a curated list of tools: a skill quotes
+ tool names, gate ids, rubric criteria, ``fix_hint`` field names, uniform names the
+ glsl contract defines, lint rule ids and battery names, and every one of those is a
+ checkable claim about something that exists. Curating them by hand would mean the
+ author of the next vocabulary has to remember to extend the test; asking "does this
+ word occur anywhere in what we ship" needs no maintenance and still fails loudly the
+ day a name is renamed out of the codebase.
+ """
+ words: set[str] = set()
+ for root, exts in ((HARNESS / "codeverse", {".py", ".md", ".j2", ".yaml", ".yml", ".toml"}),
+ (HARNESS / "bench", {".py", ".yaml", ".yml"}),
+ (HARNESS / "runtime_js", {".js", ".mjs", ".glsl", ".json"})):
+ if not root.is_dir():
+ continue
+ for p in root.rglob("*"):
+ if (not p.is_file() or p.suffix not in exts
+ or "node_modules" in p.parts or "__pycache__" in p.parts
+ or p.parent.name.startswith("cv3d-") or "_claims" in p.parts
+ or p.parent.parent.name.startswith("cv3d-")):
+ continue
+ words |= set(re.findall(r"[A-Za-z_][A-Za-z0-9_]*", p.read_text(errors="ignore")))
+ out = HARNESS / "bench" / "out" # battery names a bundle cites as its evidence
+ if out.is_dir():
+ words |= {d.name for d in out.iterdir() if d.is_dir()}
+ return words
+
+
+LIVE_VOCABULARY = _live_vocabulary()
+
+#: snake_case names that are NOT ours, with who owns them. Extend deliberately.
+FOREIGN_CALLS = {
+ # GLSL / three.js / bpy builtins a bundle quotes while teaching that language
+ "texture", "smoothstep", "fract", "mix", "clamp", "dot", "cross", "normalize", "length",
+ "abs", "max", "min", "mod", "pow", "sin", "cos", "step", "reflect", "refract",
+ "transform_apply", "primitive_cube_add", "ensure_lookup_table", "hide_set", "update",
+ "len", "setup", "range", "print", "float", "int",
+ # illustrative names inside a bundle's own worked example
+ "left_wheel", "right_wheel",
+ # batteries whose results are cited as evidence but whose output dir is gitignored
+ "graphics_v1_flash", "graphics_v2_flash", "scenes_v1_flash",
+}
+
+
+def _squash(text: str) -> str:
+ """One-line, lowercase — a section name wrapped across two markdown lines is one name."""
+ return " ".join(text.lower().split())
+
+
+def _prose(text: str) -> str:
+ """The body with fenced code removed — a define inside a shader is not our vocabulary."""
+ return _FENCE.sub("", text)
+
+
[email protected]("doc", DOCS, ids=[str(p.relative_to(skills_dir())) for p in DOCS])
+def test_every_harness_name_a_bundle_quotes_is_still_a_live_name(doc: Path):
+ """`check_connectivity` must not become a name only the skill remembers.
+
+ Wider than the tool registry on purpose: a rubric criterion a skill tells the agent to
+ optimise is as much a live name as a tool, and rubric criteria get renamed.
+ """
+ text = _prose(doc.read_text())
+ named = {t.strip("`").split("(")[0].strip() for t in _INLINE_CODE.findall(text)}
+ named = {t for t in named if _HARNESS_SHAPED.match(t)}
+ unknown = sorted(named - LIVE_VOCABULARY - FOREIGN_CALLS)
+ assert unknown == [], (
+ f"{doc.name} quotes {unknown}, which no tool, gate, rubric, criterion, language or "
+ f"contract entry point answers to today")
+
+
[email protected]("doc", DOCS, ids=[str(p.relative_to(skills_dir())) for p in DOCS])
+def test_every_gate_kind_a_bundle_names_is_a_live_kind(doc: Path):
+ """A reworded gate that changes a slug must break here, not silently unroute a sheet."""
+ slugs = set(_GATE_SLUG.findall(_prose(doc.read_text())))
+ unknown = sorted(slugs - LIVE_KINDS)
+ assert unknown == [], f"{doc.name} names gate kind(s) registry.py does not define: {unknown}"
+
+
[email protected]("doc", DOCS, ids=[str(p.relative_to(skills_dir())) for p in DOCS])
+def test_every_constant_a_bundle_names_in_prose_is_ours_or_declared_foreign(doc: Path):
+ toks = set(_CONST.findall(_prose(doc.read_text())))
+ unknown = sorted(t for t in toks if t not in LIVE_CONSTANTS and t not in FOREIGN)
+ assert unknown == [], (
+ f"{doc.name} names {unknown}: either the constant moved, or add it to FOREIGN "
+ f"with the namespace that owns it")
+
+
[email protected]("doc", DOCS, ids=[str(p.relative_to(skills_dir())) for p in DOCS])
+def test_every_switch_a_bundle_names_is_a_live_switch(doc: Path):
+ from codeverse.tracks.plan_features import LIVE_SWITCHES
+
+ live = {s if isinstance(s, str) else getattr(s, "env", "") for s in LIVE_SWITCHES}
+ live |= {n for n in LIVE_CONSTANTS if n.startswith("CV3D_")}
+ live |= {"CV3D_SKILLS", "CV3D_SKILLS_MAX", "CV3D_SKILLS_UNVERIFIED", "CV3D_SKILLS_DIR",
+ "CV3D_MAX_IN_FLIGHT"}
+ named = set(_ENV.findall(doc.read_text()))
+ assert named <= live, f"{doc.name} names dead switch(es): {sorted(named - live)}"
+
+
[email protected]("doc", DOCS, ids=[str(p.relative_to(skills_dir())) for p in DOCS])
+def test_every_sibling_skill_a_bundle_names_exists(doc: Path):
+ named = set(re.findall(r"\bcv3d-[a-z0-9-]+\b", doc.read_text()))
+ have = {d.name for d in BUNDLES} | set(ROUTED_SKILLS)
+ assert named <= have, f"{doc.name} points at missing skill(s): {sorted(named - have)}"
+
+
+def _languages_of(skill_name: str) -> set[str]:
+ out: set[str] = set()
+ for r in ROUTES:
+ if r.skill == skill_name:
+ out |= set(r.languages)
+ return out
+
+
[email protected]("s", SKILLS, ids=[s.name for s in SKILLS])
+def test_every_cookbook_section_a_skill_names_exists_in_that_cookbook(s):
+ """T4's other half: a skill may point at the cookbook, never invent a heading."""
+ from codeverse.prompts import load_text
+ from codeverse.spatial.cookbook_tool import _PROMPT_DIR, split_sections
+
+ text = "\n".join([s.body] + [p.read_text() for p in sorted(s.dir.rglob("references/*.md"))])
+ wanted = set(_COOKBOOK_SECTION.findall(text))
+ if not wanted:
+ pytest.skip("names no cookbook section")
+ langs = _languages_of(s.name) or {"blender"}
+ titles: set[str] = set()
+ for lang in langs:
+ d = _PROMPT_DIR.get(lang, lang)
+ try:
+ md = load_text(f"{d}/cookbook.md")
+ except (FileNotFoundError, OSError):
+ continue
+ titles |= {sec.title.lower() for sec in split_sections(md)}
+ for want in sorted(_squash(w) for w in wanted):
+ assert any(want in _squash(t) for t in titles), (
+ f"{s.name} sends the agent to read_cookbook(section={want!r}), "
+ f"which no cookbook of {sorted(langs)} has")
diff --git a/harness/tests/skills/test_library.py b/harness/tests/skills/test_library.py
index ef7788fd..44a5b8b2 100644
--- a/harness/tests/skills/test_library.py
+++ b/harness/tests/skills/test_library.py
@@ -18,7 +18,7 @@ from pathlib import Path
import pytest
from codeverse.skills import bundle_dirs, iter_skills, skills_dir, validate_bundle
-from codeverse.skills.claims import check_claims, claim_values
+from codeverse.skills.claims import check_claims, claim_bases, load_claims
from codeverse.skills.loader import BODY_MAX_LINES, BODY_MAX_TOKENS
from codeverse.skills.registry import ROUTED_SKILLS, ROUTES
@@ -73,18 +73,37 @@ def test_an_evidence_thin_bundle_says_so(s):
assert s.metadata.get("evidence_note"), f"{s.name}: a non-measured bundle must say why in metadata"
-def test_co_routing_skills_never_disagree_about_a_number():
- """T3: astra3d checked one hand-picked pair; this checks the whole co-routing graph."""
- by_skill = {s.name: claim_values(s.name) for s in SKILLS}
- pairs: set[tuple[str, str]] = set()
- for a in by_skill:
- for b in by_skill:
- if a < b and _can_co_route(a, b):
- pairs.add((a, b))
- for a, b in sorted(pairs):
- shared = set(by_skill[a]) & set(by_skill[b])
- for key in sorted(shared):
- assert by_skill[a][key] == by_skill[b][key], f"{a} and {b} disagree about {key}"
+def test_no_two_skills_anywhere_point_one_claim_key_at_different_numbers():
+ """T3: astra3d checked one hand-picked pair by hand; this checks the whole library.
+
+ Compared on the PRE-SCALE value, not the rendered text: "1 cm" and "0.01" metres are
+ the same tolerance honestly quoted in two units, and failing that pair would have
+ taught the next author to delete the claim rather than fix a contradiction.
+ """
+ by_key: dict[str, dict[str, tuple[str, object]]] = defaultdict(dict)
+ for s in SKILLS:
+ for key, base in claim_bases(s.name).items():
+ by_key[key][s.name] = base
+ for key, owners in sorted(by_key.items()):
+ targets = {t for t, _ in owners.values()}
+ values = {repr(v) for _, v in owners.values()}
+ assert len(targets) == 1, f"{key} is pinned to {sorted(targets)} by {sorted(owners)}"
+ assert len(values) == 1, f"{key} resolves to {sorted(values)} across {sorted(owners)}"
+
+
+def test_co_routing_skills_render_a_shared_number_the_same_way():
+ """A weaker but still useful rule for skills that can land in ONE session together:
+ if they chose the same units for a shared key, the sentence must read the same."""
+ rows = {s.name: {r["key"]: r for r in load_claims(s.name) if r.get("key")} for s in SKILLS}
+ for a in sorted(rows):
+ for b in sorted(rows):
+ if a >= b or not _can_co_route(a, b):
+ continue
+ for key in sorted(set(rows[a]) & set(rows[b])):
+ ra, rb = rows[a][key], rows[b][key]
+ if (ra.get("scale"), ra.get("format")) != (rb.get("scale"), rb.get("format")):
+ continue
+ assert ra.get("text") == rb.get("text"), f"{a} and {b} disagree about {key}"
def _can_co_route(a: str, b: str) -> bool: