Skip to content

Official Codebase of 3DCodeVerse

Python

skills: spec compliance against the reference validator, routing properties, budget and corpus recomputation — and the live smoke now actually runs

gaoypengc7ec610b47bc
diff --git a/harness/codeverse/skills/cv3d-bbox-contract/SKILL.md b/harness/codeverse/skills/cv3d-bbox-contract/SKILL.md
index 439e3ab7..3ebd5a98 100644
--- a/harness/codeverse/skills/cv3d-bbox-contract/SKILL.md
+++ b/harness/codeverse/skills/cv3d-bbox-contract/SKILL.md
@@ -4,7 +4,7 @@ description: Hit the plan's dimensions on the first build. Use whenever this ses
license: Apache-2.0
metadata:
evidence: measured
- verified: 2026-08-25
+ verified: "2026-08-25"
corpus: bench/out, 142 graded records with gate artefacts, recomputed 2026-08-25
---
diff --git a/harness/codeverse/skills/cv3d-cadquery-forms/SKILL.md b/harness/codeverse/skills/cv3d-cadquery-forms/SKILL.md
index 071ab85d..a015d6ac 100644
--- a/harness/codeverse/skills/cv3d-cadquery-forms/SKILL.md
+++ b/harness/codeverse/skills/cv3d-cadquery-forms/SKILL.md
@@ -6,7 +6,7 @@ compatibility: CadQuery 2.8 on OCP; one module src/model.py exporting a module-l
metadata:
evidence: inherited-unverified
evidence_note: 'Every kernel rule below was probed against the CadQuery 2.8.0 installed in this repo on 2026-08-25 and cross-checked with the cadquery lint, the run_cq wrapper and the language contract. What is NOT available is score evidence. This harness has ZERO graded cadquery bench runs, so nothing here is ranked by measured defect frequency the way the blender and urdf bundles are. Routed off by default until cadquery has 20 graded runs.'
- verified: 2026-08-25
+ verified: "2026-08-25"
---
# CadQuery: forms, and the kernel rules that gate them
diff --git a/harness/codeverse/skills/cv3d-opengl-pipeline/SKILL.md b/harness/codeverse/skills/cv3d-opengl-pipeline/SKILL.md
index 0c4a14dc..07b3b708 100644
--- a/harness/codeverse/skills/cv3d-opengl-pipeline/SKILL.md
+++ b/harness/codeverse/skills/cv3d-opengl-pipeline/SKILL.md
@@ -4,9 +4,9 @@ description: Write a raw OpenGL / moderngl program that renders what the brief a
license: Apache-2.0
compatibility: OpenGL 3.3 core through moderngl, headless EGL; one module src/program.py, GLSL 330 core inline or in src/*.glsl.
metadata:
- evidence: measured
+ evidence: mixed
evidence_note: 'Thin but real, and labelled so in the body. 5 graded opengl_python runs across graphics_v1_flash and graphics_v2_flash, 10 judged rounds, recomputed 2026-08-25. The API facts come from the runtime wrapper and the lint, not from the corpus.'
- verified: 2026-08-25
+ verified: "2026-08-25"
---
# The moderngl pipeline, and the two things that actually sink these runs
diff --git a/harness/codeverse/skills/cv3d-scene-lighting/SKILL.md b/harness/codeverse/skills/cv3d-scene-lighting/SKILL.md
index 73bf1a70..8cef2268 100644
--- a/harness/codeverse/skills/cv3d-scene-lighting/SKILL.md
+++ b/harness/codeverse/skills/cv3d-scene-lighting/SKILL.md
@@ -6,7 +6,7 @@ compatibility: three r0.182, headless WebGL; renderer fixed at ACES Filmic tone
metadata:
evidence: mixed
evidence_note: 'Thresholds and renderer settings are read from live code, namely spatial/frame_metrics.py and runtime_js/lib/browser/renderer.js, and the luminance figures are computed from those settings. The corpus half is thin and labelled as such, being 4 scene_threejs runs in scenes_v1_flash mined 2026-08-25, so it is quoted as examples and never as a rate.'
- verified: 2026-08-25
+ verified: "2026-08-25"
---
# Lighting a scene the frame gate will accept
diff --git a/harness/codeverse/skills/loader.py b/harness/codeverse/skills/loader.py
index 3f993a38..45b2363c 100644
--- a/harness/codeverse/skills/loader.py
+++ b/harness/codeverse/skills/loader.py
@@ -73,6 +73,20 @@ def _as_str_map(value: object, where: str) -> dict[str, str]:
return out
+def nonstring_metadata(fm: dict) -> list[str]:
+ """metadata values YAML did not hand back as strings.
+
+ An unquoted ``verified: 2026-08-25`` is a YAML *date*, not a string, and the spec says
+ metadata is string to string. ``_as_str_map`` coerces so a run never dies of it; the
+ validator reports it so a bundle is not shipped depending on our leniency.
+ """
+ meta = fm.get("metadata")
+ if not isinstance(meta, dict):
+ return []
+ return [f"metadata.{k} is a {type(v).__name__}, not a string — quote it (the spec allows string→string)"
+ for k, v in meta.items() if not isinstance(v, str) and v is not None]
+
+
def parse_skill(path: Path) -> Skill:
"""Read ``<dir>/SKILL.md`` into a :class:`Skill`, or raise :class:`SkillError`."""
path = Path(path)
@@ -155,6 +169,11 @@ def validate_bundle(bundle: Path) -> list[str]:
issues.append(f"metadata.evidence must be one of {list(EVIDENCE_LEVELS)}, got {skill.evidence!r}")
if not skill.metadata.get("verified"):
issues.append("metadata.verified (ISO date the claims were last checked) is missing")
+ try:
+ fm_text, _ = split_frontmatter((bundle / SKILL_FILE).read_text(encoding="utf-8"))
+ issues += nonstring_metadata(yaml.safe_load(fm_text) or {})
+ except (OSError, SkillError, yaml.YAMLError): # already reported by parse_skill above
+ pass
refs = bundle / REFERENCES_DIR
if refs.is_dir():
for p in refs.iterdir():
@@ -167,4 +186,5 @@ def validate_bundle(bundle: Path) -> list[str]:
__all__ = ["BODY_MAX_LINES", "BODY_MAX_TOKENS", "DESCRIPTION_MAX", "NAME_MAX", "NAME_RE", "REFERENCES_DIR",
- "SKILL_FILE", "SPEC_KEYS", "SkillError", "parse_skill", "split_frontmatter", "validate_bundle"]
+ "SKILL_FILE", "SPEC_KEYS", "SkillError", "nonstring_metadata", "parse_skill",
+ "split_frontmatter", "validate_bundle"]
diff --git a/harness/tests/skills/conftest.py b/harness/tests/skills/conftest.py
index b95286ba..7bc2a3f3 100644
--- a/harness/tests/skills/conftest.py
+++ b/harness/tests/skills/conftest.py
@@ -19,7 +19,7 @@ description: {description}
license: Apache-2.0
metadata:
evidence: {evidence}
- verified: 2026-08-25
+ verified: "2026-08-25"
---
# {name}
diff --git a/harness/tests/skills/test_budget.py b/harness/tests/skills/test_budget.py
new file mode 100644
index 00000000..3d1da556
--- /dev/null
+++ b/harness/tests/skills/test_budget.py
@@ -0,0 +1,85 @@
+"""What the library costs, measured against the SHIPPED bundles and capped.
+
+WHY a file of its own, and why it uses the real library: ``test_materialize_prompting``
+prices a synthetic fixture whose descriptions are one line long. The real ones use the
+spec's 1024-char allowance, because a description is the only text a CLI matches on —
+and quoting all fourteen of them cost 780 tokens for a five-skill session, against a
+300-token design budget, until the index started quoting the first clause instead.
+A cost regression is invisible in a fixture and expensive in production, so the numbers
+below are re-measured over every session the router can produce.
+
+docs/COST.md §4: 39.3% of the agent bill is spent from turn 20 onwards and 19 of 33 long
+sessions ended `budget`. Everything here is per turn, not per session.
+"""
+
+from __future__ import annotations
+
+import itertools
+
+import pytest
+
+from codeverse.skills import all_skills, bundle_dirs, select, skills_dir
+from codeverse.skills.loader import BODY_MAX_TOKENS
+from codeverse.skills.prompting import NATIVE_LOADERS, index_block, index_tokens
+
+pytestmark = pytest.mark.skipif(not bundle_dirs(), reason=f"no bundles in {skills_dir()} yet")
+
+LIBRARY = all_skills()
+
+#: the api-agent index is the only index we write; five skills must fit in this
+API_INDEX_TOKENS_MAX = 400
+API_INDEX_BYTES_MAX = 2048
+#: a native loader gets one sentence — its own loader writes the real index
+NATIVE_INDEX_TOKENS_MAX = 60
+
+TRACKS = ("static_object", "articulated_object", "scene", "graphics")
+LANGUAGES = ("blender", "cadquery", "threejs", "urdf_blender", "scene_threejs",
+ "glsl_shader", "opengl_python")
+KINDS = ("baseline", "part", "detail", "refine", "rebuild", "repair", "env", "zone", "compose")
+#: one of everything, so the cap is always the binding constraint
+LOUD_SIGNALS = {"multi_part": True, "has_instances": True, "has_symmetry": True,
+ "has_joints": True, "has_custom_shader": True, "has_assemblies": True, "n_parts": 9}
+ALL_FINDINGS = ["connectivity/interpenetration", "contract/part_bbox", "contract/instance_bbox",
+ "joint_sweep/link_overlap", "motion_direction/wrong_axis", "scene_frames/dark_or_flat",
+ "gl_frames/motion_or_detail", "lint/part_not_imported", "shader/compile_or_binding",
+ "scene_frames/camera_placement"]
+
+
+def _every_session():
+ for track, language, kind in itertools.product(TRACKS, LANGUAGES, KINDS):
+ for findings in ((), ALL_FINDINGS):
+ for unverified in (False, True):
+ yield track, language, kind, [
+ s.skill for s in select(track, language, kind, signals=LOUD_SIGNALS,
+ findings=findings, library=LIBRARY, max_skills=5,
+ allow_unverified=unverified)]
+
+
+def test_the_api_agent_index_stays_inside_its_budget_with_the_whole_library_installed():
+ worst = max(_every_session(), key=lambda row: index_tokens(row[3], "api-agent"))
+ tokens = index_tokens(worst[3], "api-agent")
+ assert tokens <= API_INDEX_TOKENS_MAX, f"{worst[:3]} costs {tokens} tokens of message 0"
+ assert len(index_block(worst[3], "api-agent").encode()) <= API_INDEX_BYTES_MAX
+
+
[email protected]("kind", NATIVE_LOADERS)
+def test_a_native_loader_is_never_handed_a_second_index(kind: str):
+ for track, language, round_kind, skills in _every_session():
+ text = index_block(skills, kind)
+ assert index_tokens(skills, kind) <= NATIVE_INDEX_TOKENS_MAX
+ for s in skills:
+ assert s.name not in text, f"{kind} would index {s.name} twice ({track}/{language}/{round_kind})"
+
+
+def test_no_body_can_blow_a_session_on_its_own():
+ for name, skill in sorted(LIBRARY.items()):
+ assert skill.body_tokens <= BODY_MAX_TOKENS, f"{name}: {skill.body_tokens} tokens"
+
+
+def test_reading_the_whole_routed_set_is_priced_and_bounded():
+ """The worst thing an agent can do is read all five. It must still be affordable."""
+ worst = max(_every_session(), key=lambda row: sum(s.body_tokens for s in row[3]))
+ total = sum(s.body_tokens for s in worst[3])
+ assert total <= 5 * BODY_MAX_TOKENS
+ # ~$0.0021/1k in, gemini-3.7-flash: five bodies is cents, not dollars
+ assert total * 2.1e-6 < 0.03, f"{worst[:3]} would cost ${total * 2.1e-6:.4f} to read fully"
diff --git a/harness/tests/skills/test_corpus.py b/harness/tests/skills/test_corpus.py
new file mode 100644
index 00000000..c6c90326
--- /dev/null
+++ b/harness/tests/skills/test_corpus.py
@@ -0,0 +1,143 @@
+"""Recompute, from bench/out, the corpus a bundle claims to have been mined from.
+
+WHY this exists: "fabricated authority on thin evidence" is the risk the whole evidence
+mechanism was built against. ``metadata.evidence`` is a label an author types; this is
+the only thing that checks it. cadquery and threejs have ZERO graded runs, and a bundle
+that quietly said otherwise would be exactly the confident-prose-with-no-data the corpus
+was supposed to replace.
+
+WHY the assertions are one-sided rather than "within 20%": ``bench/out`` is a live
+directory that sibling batteries write into while this test runs, so the true count only
+ever grows after a mining date. Two invariants survive that and still catch what matters:
+
+ fabrication a claim may never exceed what the directory actually holds — you cannot
+ have mined 200 runs out of 139.
+ authority ``evidence: measured`` needs a language with at least 20 graded runs behind
+ it; below that the honest label is ``mixed`` (or ``inherited-unverified``
+ at zero), and the bundle owes an ``evidence_note``.
+ provenance a percentage presented as a corpus rate must name its battery, n or date
+ within a few sentences, so a reader can re-derive it.
+
+What this deliberately does NOT do is assert a claim to within 20% of the live count. A
+body's "47 graded blender runs" is a SLICE (one battery, one track, one round kind) and
+bench/out holds 139 blender runs overall; without each claim declaring the query that
+produced it, a tight recomputation compares two different populations and fails honestly
+written prose. Declaring those queries the way ``_claims/*.toml`` declares constants is
+the open item — see docs/SKILLS.md.
+
+Skipped when ``bench/out`` is absent (a fresh checkout, and CI), which is why it is also
+marked slow: it walks a few hundred records.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+from collections import Counter
+from pathlib import Path
+
+import pytest
+
+from codeverse.skills import bundle_dirs, iter_skills, skills_dir
+
+BUNDLES = bundle_dirs()
+SKILLS = list(iter_skills()) if BUNDLES else []
+BENCH_OUT = Path(__file__).resolve().parents[2] / "bench" / "out"
+
+pytestmark = [
+ pytest.mark.slow,
+ pytest.mark.skipif(not BUNDLES, reason=f"no bundles in {skills_dir()} yet"),
+ pytest.mark.skipif(not BENCH_OUT.is_dir(), reason="no bench/out in this checkout"),
+]
+
+#: "102 graded blender runs", "23 graded urdf_blender runs", "9 graded glsl runs"
+_CLAIM = re.compile(r"\b(\d+)\s+graded\s+([a-z_]+?)(?:_blender)?\s*runs?\b", re.I)
+#: shorthand a body may use for a language id
+_ALIAS = {"urdf": "urdf_blender", "glsl": "glsl_shader", "opengl": "opengl_python",
+ "scene": "scene_threejs", "blender": "blender", "cadquery": "cadquery",
+ "threejs": "threejs", "urdf_blender": "urdf_blender", "glsl_shader": "glsl_shader",
+ "opengl_python": "opengl_python", "scene_threejs": "scene_threejs"}
+#: a percentage is a MEASUREMENT (and needs provenance) when it carries a decimal, or
+#: sits next to corpus words. "a 5-15 % taper" is a design heuristic, not a statistic.
+_MEASURED_PCT = re.compile(
+ r"\d{1,3}(?:\.\d)?\s?%[^.\n]{0,40}\b(?:of runs|runs|corpus|battery|of the corpus)\b"
+ r"|\b(?:runs|corpus|battery|fired|deviates)\b[^.\n]{0,40}\d{1,3}(?:\.\d)?\s?%")
+_PROVENANCE = re.compile(r"(bench/out|battery|batteries|\bn\s?=\s?\d+|20\d\d-\d\d-\d\d|\bof\s+\d+\s+runs\b"
+ r"|\b\d+\s+(?:graded|run|runs|records?)\b)")
+
+
+def _graded_by_language() -> Counter:
+ """Graded runs that also carry gate artefacts, per language, as of right now."""
+ out: Counter = Counter()
+ for dirpath, _dirs, files in os.walk(BENCH_OUT, followlinks=True):
+ if "record.json" not in files:
+ continue
+ try:
+ d = json.loads((Path(dirpath) / "record.json").read_text())
+ except (OSError, json.JSONDecodeError):
+ continue
+ lang = (d.get("spec") or {}).get("language")
+ has_gates = any(r.get("gates") for r in (d.get("rounds") or []))
+ if lang and has_gates and d.get("final_score") is not None:
+ out[lang] += 1
+ return out
+
+
+LIVE = _graded_by_language()
+
+
+def test_the_corpus_is_big_enough_to_be_worth_mining():
+ assert sum(LIVE.values()) >= 50, f"bench/out only holds {sum(LIVE.values())} graded runs"
+
+
[email protected]("s", SKILLS, ids=[s.name for s in SKILLS])
+def test_no_bundle_claims_more_runs_than_bench_out_holds(s):
+ text = s.body + " " + " ".join(f"{k}: {v}" for k, v in s.metadata.items())
+ for count, raw in _CLAIM.findall(text):
+ lang = _ALIAS.get(raw.lower())
+ if lang is None:
+ continue
+ have = LIVE.get(lang, 0)
+ assert int(count) <= have, (
+ f"{s.name} claims {count} graded {lang} runs; bench/out holds {have} today")
+
+
[email protected]("s", SKILLS, ids=[s.name for s in SKILLS])
+def test_a_bundles_evidence_label_matches_the_data_behind_it(s):
+ """`measured` is a claim about n, not a mood. Zero graded runs is unverified."""
+ from codeverse.skills.model import EVIDENCE_INHERITED, EVIDENCE_MEASURED
+ from codeverse.skills.registry import ROUTES
+
+ langs = {lang for r in ROUTES if r.skill == s.name for lang in r.languages}
+ if not langs:
+ return # a track-wide sheet: no language to count
+ best = max(LIVE.get(lang, 0) for lang in langs)
+ if best == 0:
+ assert s.evidence == EVIDENCE_INHERITED, (
+ f"{s.name} routes only to {sorted(langs)}, which have no graded runs at all")
+ elif best < 20:
+ assert s.evidence != EVIDENCE_MEASURED, (
+ f"{s.name}'s best language has {best} graded runs — `measured` overstates it, "
+ f"use `mixed` with an evidence_note")
+ if s.evidence != EVIDENCE_MEASURED:
+ assert s.metadata.get("evidence_note"), f"{s.name}: {s.evidence} owes an evidence_note"
+
+
[email protected]("s", SKILLS, ids=[s.name for s in SKILLS])
+def test_the_verified_date_is_a_real_past_date(s):
+ """The only staleness signal that survives a growing corpus."""
+ import datetime as dt
+
+ raw = s.metadata.get("verified", "")
+ when = dt.date.fromisoformat(str(raw))
+ assert when <= dt.date.today(), f"{s.name}: metadata.verified {raw} is in the future"
+
+
[email protected]("s", SKILLS, ids=[s.name for s in SKILLS])
+def test_every_measured_percentage_carries_its_provenance(s):
+ """A reader must be able to re-derive the number without guessing which battery."""
+ for m in _MEASURED_PCT.finditer(s.body):
+ window = s.body[max(0, m.start() - 300): m.end() + 300]
+ assert _PROVENANCE.search(window), (
+ f"{s.name} quotes {m.group(0)!r} with no battery / n / date within 300 chars")
diff --git a/harness/tests/skills/test_library.py b/harness/tests/skills/test_library.py
index 44a5b8b2..e6f4aeb6 100644
--- a/harness/tests/skills/test_library.py
+++ b/harness/tests/skills/test_library.py
@@ -10,8 +10,6 @@ add a test for each of those.
from __future__ import annotations
import re
-import shutil
-import subprocess
from collections import defaultdict
from pathlib import Path
@@ -134,32 +132,3 @@ def test_every_routed_skill_has_a_bundle_once_the_library_is_complete():
def test_no_bundle_exists_that_no_route_can_ever_attach():
orphans = [d.name for d in BUNDLES if d.name not in ROUTED_SKILLS]
assert orphans == [], f"unroutable bundles pay the index and never help: {orphans}"
-
-
[email protected](shutil.which("skills-ref") is None,
- reason="the reference validator is not installed (pip install skills-ref)")
[email protected]("d", BUNDLES, ids=[d.name for d in BUNDLES])
-def test_the_reference_validator_accepts_the_bundle(d: Path):
- """T1: our loader implements the spec, but the spec's own validator is the arbiter."""
- p = subprocess.run(["skills-ref", "validate", str(d)], capture_output=True, text=True, check=False)
- assert p.returncode == 0, p.stdout + p.stderr
-
-
-def test_every_corpus_statistic_a_skill_quotes_is_still_true():
- """T8: a percentage in a body must be within 20% of what bench/out says today."""
- out = Path(__file__).resolve().parents[2] / "bench" / "out"
- if not out.is_dir():
- pytest.skip("no bench/out in this checkout")
- quoted = defaultdict(list)
- for s in SKILLS:
- for m in re.finditer(r"(\d{1,3}(?:\.\d)?)\s?%", s.body):
- quoted[s.name].append(float(m.group(1)))
- if not quoted:
- pytest.skip("no corpus percentages quoted yet")
- # The recomputation lives with the corpus tooling; here we only assert the bodies
- # carry their provenance, so a reader can check the number without guessing.
- for name, values in quoted.items():
- body = next(s.body for s in SKILLS if s.name == name)
- assert re.search(r"(bench/out|battery|n\s?=\s?\d+|20\d\d-\d\d-\d\d)", body), (
- f"{name} quotes {values} with no battery / n / date beside it")
diff --git a/harness/tests/skills/test_live_discovery.py b/harness/tests/skills/test_live_discovery.py
index 29a3a945..d10bc54e 100644
--- a/harness/tests/skills/test_live_discovery.py
+++ b/harness/tests/skills/test_live_discovery.py
@@ -12,7 +12,6 @@ and opened the bundle, ``atime > mtime`` on its ``references/`` file.
from __future__ import annotations
-import shutil
from pathlib import Path
import pytest
@@ -30,14 +29,21 @@ PROMPT = ("Your workspace has a skill installed whose description matches this t
"directory, then reply with the single magic word the reference file contains. "
"Do not write any file.")
MAGIC = "ORTHOGONAL-KUMQUAT"
-BACKENDS = [("gemini-cli", "gemini-3.7-flash"), ("codex", "gpt-5.2-codex"),
+#: the model each CLI is driven with here. ``codex`` must name a model the local login
+#: is entitled to — a 400 "not supported when using Codex with a ChatGPT account" is an
+#: auth problem, not a discovery problem, and would read as a discovery failure.
+BACKENDS = [("gemini-cli", "gemini-3.7-flash"), ("codex", "gpt-5.6-luna"),
("claude-code", "claude-opus-5"), ("agy", "gemini-3.7-flash")]
@pytest.mark.parametrize("kind, model", BACKENDS, ids=[k for k, _ in BACKENDS])
def test_the_cli_discovers_and_opens_a_materialised_skill(tmp_path: Path, kind: str, model: str):
- if shutil.which(kind) is None:
- pytest.skip(f"{kind} is not installed here")
+ # the binary is NOT named after the agent kind (`gemini-cli` runs `gemini`,
+ # `claude-code` runs `claude`), so ask the adapter, which owns that mapping
+ agent = get_coding_agent(f"{kind}:{model}")
+ ok, why = agent.available()
+ if not ok:
+ pytest.skip(f"{kind} unavailable: {why}")
ws = tmp_path / "ws"
(ws / "src").mkdir(parents=True)
for name in ("AGENTS.md", "GEMINI.md", "CLAUDE.md"):
@@ -56,10 +62,6 @@ def test_the_cli_discovers_and_opens_a_materialised_skill(tmp_path: Path, kind:
materialize_skills(ws, [skill])
write_index(ws, index_block([skill], kind))
- agent = get_coding_agent(f"{kind}:{model}")
- ok, why = agent.available()
- if not ok:
- pytest.skip(f"{kind} unavailable: {why}")
res = agent.run(AgentJob(workspace=str(ws), prompt=PROMPT, spatial_tools=False,
max_turns=8, timeout_s=300, label="skills_live"))
diff --git a/harness/tests/skills/test_packaging.py b/harness/tests/skills/test_packaging.py
index 1f56cb72..1b0da453 100644
--- a/harness/tests/skills/test_packaging.py
+++ b/harness/tests/skills/test_packaging.py
@@ -50,10 +50,18 @@ def test_a_built_wheel_actually_contains_the_bundles(tmp_path: Path):
assert r.returncode == 0, r.stdout + r.stderr
whl = next(out.glob("*.whl"))
names = set(zipfile.ZipFile(whl).namelist())
- want = PROBE if made_probe else existing[0]
- assert f"codeverse/skills/{want}/SKILL.md" in names, sorted(n for n in names if "skills" in n)[:20]
- refs = [n for n in names if n.startswith(f"codeverse/skills/{want}/references/")]
- assert refs, "references/ must ship too — it is the depth probe"
+ want = [PROBE] if made_probe else sorted(existing)
+ for name in want:
+ assert f"codeverse/skills/{name}/SKILL.md" in names, (
+ f"{name} is in the source tree and NOT in the wheel: "
+ f"{sorted(n for n in names if 'skills' in n)[:20]}")
+ refs = [n for n in names if n.startswith(f"codeverse/skills/{name}/references/")]
+ assert refs, f"{name}: references/ must ship too — it is the depth probe"
+ if not made_probe:
+ # every claims file too: `3dcv skills validate` runs against an installed wheel
+ claims = sorted((HARNESS / "codeverse" / "skills" / "_claims").glob("*.toml"))
+ for c in claims:
+ assert f"codeverse/skills/_claims/{c.name}" in names, f"{c.name} missing from the wheel"
assert not any(n.endswith(".pyc") for n in names if "skills" in n)
finally:
if made_probe:
diff --git a/harness/tests/skills/test_routing_property.py b/harness/tests/skills/test_routing_property.py
new file mode 100644
index 00000000..7abb8833
--- /dev/null
+++ b/harness/tests/skills/test_routing_property.py
@@ -0,0 +1,171 @@
+"""Routing over the WHOLE input space, against the REAL library — invariants, not examples.
+
+``test_router`` checks the rows one at a time with a synthetic library. This file runs
+the router over the full cross product of the inputs a run can actually present — every
+track x language x round kind x plan-signal combination, and every gate finding kind the
+corpus produces — and asserts the four properties that must hold for all of them:
+
+ determinism the same input always gives the same ordered list
+ the cap no input ever exceeds ``max_skills``
+ totality a junk input degrades to an empty list, never an exception
+ relevance a gate-fired sheet outranks a standing one, and the routed set is
+ explainable — every selection names the row that put it there
+
+The library is the shipped one, not a fixture: the property that matters is that THESE
+fourteen bundles route sanely, and a bundle whose evidence is inherited-unverified must
+stay out unless the switch says otherwise.
+"""
+
+from __future__ import annotations
+
+import itertools
+import random
+
+import pytest
+
+from codeverse.skills import all_skills, bundle_dirs, select, skills_dir
+from codeverse.skills.model import EVIDENCE_INHERITED
+from codeverse.skills.registry import QUIET_KINDS, ROUTED_SKILLS, ROUTES
+from codeverse.skills.router import SIGNAL_KEYS
+
+pytestmark = pytest.mark.skipif(not bundle_dirs(), reason=f"no bundles in {skills_dir()} yet")
+
+LIBRARY = all_skills()
+
+TRACKS = ("static_object", "articulated_object", "scene", "graphics")
+LANGUAGES = ("blender", "cadquery", "threejs", "urdf_blender", "scene_threejs",
+ "glsl_shader", "opengl_python")
+KINDS = ("baseline", "part", "detail", "refine", "rebuild", "repair", "env", "zone",
+ "compose", "asset", "asset_fix", "reference")
+#: the boolean plan signals the table may test (n_parts / joint_types are derived)
+FLAGS = tuple(k for k in SIGNAL_KEYS if k not in ("n_parts", "joint_types"))
+JUNK = ("", " ", "no_such_track", "STATIC_OBJECT", "static object", "1", "../etc")
+
+LIVE_KINDS = sorted({k for row in ROUTES for k in row.findings if not k.endswith("*")} |
+ {"connectivity/interpenetration", "connectivity/floating_part",
+ "connectivity/stray_islands", "contract/part_bbox", "contract/overall_bbox",
+ "joint_sweep/link_overlap", "motion_direction/wrong_axis",
+ "scene_frames/dark_or_flat", "gl_frames/motion_or_detail",
+ "lint/part_not_imported", "shader/compile_or_binding"})
+
+
+def _signal_sets():
+ """Every boolean combination, each with the n_parts it implies."""
+ for bits in itertools.product((False, True), repeat=len(FLAGS)):
+ sig = dict(zip(FLAGS, bits, strict=True))
+ sig["n_parts"] = 6 if sig["multi_part"] else 1
+ sig["joint_types"] = ["revolute"] if sig["has_joints"] else []
+ yield sig
+
+
+BASE_INPUTS = [(t, lang, kind) for t in TRACKS for lang in LANGUAGES for kind in KINDS]
+
+
[email protected]("track", TRACKS)
+def test_the_cap_and_the_ordering_hold_over_the_whole_input_space(track: str):
+ seen = 0
+ for language, kind in itertools.product(LANGUAGES, KINDS):
+ for sig in _signal_sets():
+ got = select(track, language, kind, signals=sig, library=LIBRARY, max_skills=5)
+ seen += 1
+ assert len(got) <= 5, f"{track}/{language}/{kind} routed {len(got)}"
+ assert len({s.name for s in got}) == len(got), "a skill was attached twice"
+ prios = [s.priority for s in got]
+ assert prios == sorted(prios, reverse=True), f"{track}/{language}/{kind} is out of order"
+ for s in got:
+ assert s.rules and s.reason, f"{s.name} was attached with no rule to point at"
+ assert seen == len(LANGUAGES) * len(KINDS) * 2 ** len(FLAGS)
+
+
[email protected]("track", TRACKS)
+def test_routing_is_deterministic_over_the_whole_input_space(track: str):
+ for language, kind in itertools.product(LANGUAGES, KINDS):
+ sig = {"multi_part": True, "has_instances": True, "has_custom_shader": True, "n_parts": 4}
+ a = [s.name for s in select(track, language, kind, signals=sig, library=LIBRARY)]
+ b = [s.name for s in select(track, language, kind, signals=sig, library=LIBRARY)]
+ assert a == b
+
+
[email protected]("finding", LIVE_KINDS)
+def test_every_corpus_finding_kind_routes_sanely_from_every_session(finding: str):
+ """A gate finding must never crash the router, never blow the cap, and never
+ silently outrank nothing — where a row answers it, it must come back at >= 90."""
+ answered_somewhere = False
+ for track, language, kind in BASE_INPUTS:
+ got = select(track, language, kind, findings=[finding], library=LIBRARY, max_skills=5)
+ assert len(got) <= 5
+ fired = [s for s in got if s.gate_fired]
+ for s in fired:
+ assert s.priority >= 90, f"{s.name} answered {finding} at priority {s.priority}"
+ assert finding in s.reason or any(finding.startswith(p.rstrip("*"))
+ for r in ROUTES if r.rule in s.rules for p in r.findings)
+ answered_somewhere = answered_somewhere or bool(fired)
+ assert answered_somewhere, f"{finding} is classified but no row anywhere answers it"
+
+
+def test_a_quiet_kind_stays_quiet_until_a_gate_fires():
+ for track, language in itertools.product(TRACKS, LANGUAGES):
+ for kind in QUIET_KINDS:
+ sig = {"multi_part": True, "has_instances": True, "has_custom_shader": True}
+ assert select(track, language, kind, signals=sig, library=LIBRARY) == []
+
+
[email protected]("bad", JUNK)
+def test_junk_inputs_degrade_to_empty_and_never_raise(bad: str):
+ combos = [(bad, "blender", "baseline"), ("static_object", bad, "baseline"),
+ ("static_object", "blender", bad), (bad, bad, bad)]
+ for track, language, kind in combos:
+ got = select(track, language, kind, signals={"multi_part": True}, library=LIBRARY)
+ assert isinstance(got, list) and len(got) <= 5
+ for s in got:
+ # a wildcard row may legitimately fire on a junk track; it may never invent a skill
+ assert s.name in ROUTED_SKILLS
+
+
+def test_none_and_broken_inputs_are_survivable():
+ for signals in (None, {}, {"multi_part": None}, {"nonsense": object()}):
+ assert isinstance(select("static_object", "blender", "baseline",
+ signals=signals, library=LIBRARY), list)
+ for findings in ((), None, [], ["not/a/real/kind"], ["", None]):
+ assert isinstance(select("static_object", "blender", "repair",
+ findings=findings, library=LIBRARY), list)
+ assert select("static_object", "blender", "baseline", library={}) == []
+
+
+def test_every_track_language_pair_a_run_can_present_routes_something():
+ """A pair the harness actually drives must not come back empty at baseline, or the
+ library has a hole the read-rate metric would report as apathy."""
+ real = [("static_object", "blender"), ("static_object", "cadquery"), ("static_object", "threejs"),
+ ("articulated_object", "urdf_blender"), ("scene", "scene_threejs"),
+ ("graphics", "glsl_shader"), ("graphics", "opengl_python")]
+ sig = {"multi_part": True, "n_parts": 5}
+ for track, language in real:
+ got = select(track, language, "baseline", signals=sig, library=LIBRARY,
+ allow_unverified=True)
+ assert got, f"{track}/{language} routes nothing at baseline"
+
+
+def test_an_unverified_bundle_is_off_by_default_everywhere():
+ thin = {n for n, s in LIBRARY.items() if s.evidence == EVIDENCE_INHERITED}
+ if not thin:
+ pytest.skip("every shipped bundle is measured or mixed")
+ for track, language, kind in BASE_INPUTS:
+ got = {s.name for s in select(track, language, kind, signals={"multi_part": True},
+ library=LIBRARY, findings=LIVE_KINDS)}
+ assert not (got & thin), f"{sorted(got & thin)} routed without CV3D_SKILLS_UNVERIFIED"
+
+
+def test_a_random_walk_of_mixed_findings_never_breaks_an_invariant():
+ rng = random.Random(20260825)
+ for _ in range(2000):
+ track = rng.choice(TRACKS + JUNK)
+ language = rng.choice(LANGUAGES + JUNK)
+ kind = rng.choice(KINDS + JUNK)
+ sig = {k: rng.random() < 0.5 for k in FLAGS}
+ sig["n_parts"] = rng.randint(0, 30)
+ findings = rng.sample(LIVE_KINDS, rng.randint(0, 4))
+ cap = rng.randint(0, 7)
+ got = select(track, language, kind, signals=sig, findings=findings,
+ library=LIBRARY, max_skills=cap)
+ assert len(got) <= cap
+ assert [s.priority for s in got] == sorted((s.priority for s in got), reverse=True)
diff --git a/harness/tests/skills/test_spec_compliance.py b/harness/tests/skills/test_spec_compliance.py
new file mode 100644
index 00000000..609ccc23
--- /dev/null
+++ b/harness/tests/skills/test_spec_compliance.py
@@ -0,0 +1,147 @@
+"""The open Agent Skills spec (agentskills.io, Dec 2025), checked WITHOUT our loader.
+
+WHY a second implementation instead of reusing ``parse_skill``: the loader is the thing
+under test. Four of the five agent kinds this harness drives read these directories with
+their own parsers, and a bundle our loader accepts because of a bug in our loader is a
+bundle those CLIs silently drop — invisible, because the run still passes and the read
+rate simply reports a zero. So this file re-reads the raw bytes and asserts the spec's
+own field rules, and the reference validator (``pip install skills-ref`` → the
+``agentskills`` binary) is run over every bundle when it is installed.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import shutil
+import subprocess
+from pathlib import Path
+
+import pytest
+import yaml
+
+from codeverse.skills import bundle_dirs, skills_dir
+
+BUNDLES = bundle_dirs()
+pytestmark = pytest.mark.skipif(not BUNDLES, reason=f"no bundles in {skills_dir()} yet")
+IDS = [d.name for d in BUNDLES]
+
+#: the spec's field table, transcribed from research/EXTERNAL.md §1
+NAME_RE = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$")
+NAME_MAX = 64
+DESCRIPTION_MAX = 1024
+COMPATIBILITY_MAX = 500
+BODY_MAX_LINES = 500 # the spec's own recommendation; ours is stricter (350)
+SPEC_KEYS = {"name", "description", "license", "compatibility", "metadata", "allowed-tools"}
+OPTIONAL_DIRS = {"scripts", "references", "assets"}
+
+#: `skills-ref` on PyPI installs its CLI as `agentskills`; accept either spelling
+REF_VALIDATOR = shutil.which("agentskills") or shutil.which("skills-ref")
+
+
+def _raw(d: Path) -> tuple[dict, str]:
+ text = (d / "SKILL.md").read_text(encoding="utf-8")
+ m = re.match(r"\A---[ \t]*\r?\n(.*?)\r?\n---[ \t]*\r?\n?", text, re.S)
+ assert m, f"{d.name}: SKILL.md must open with a '---' frontmatter block"
+ return yaml.safe_load(m.group(1)) or {}, text[m.end():]
+
+
[email protected]("d", BUNDLES, ids=IDS)
+def test_the_bundle_is_a_directory_holding_skill_md(d: Path):
+ assert (d / "SKILL.md").is_file(), f"{d.name}: the spec's entry point is SKILL.md"
+
+
[email protected]("d", BUNDLES, ids=IDS)
+def test_name_matches_the_character_rules_and_the_directory(d: Path):
+ fm, _ = _raw(d)
+ name = fm.get("name")
+ assert isinstance(name, str) and name, f"{d.name}: `name` is required"
+ assert 1 <= len(name) <= NAME_MAX
+ assert NAME_RE.match(name), f"{name!r}: lowercase a-z0-9 and single hyphens only"
+ assert "--" not in name, f"{name!r}: consecutive hyphens are forbidden"
+ assert not name.startswith("-") and not name.endswith("-")
+ assert name == d.name, f"{name!r} != directory {d.name!r}: every CLI keys discovery off the dir"
+
+
[email protected]("d", BUNDLES, ids=IDS)
+def test_description_is_present_bounded_and_says_what_and_when(d: Path):
+ fm, _ = _raw(d)
+ desc = fm.get("description")
+ assert isinstance(desc, str) and 1 <= len(desc) <= DESCRIPTION_MAX
+ assert re.search(r"\b(use|read|reach for)\b", desc, re.I), (
+ f"{d.name}: the description is the ONLY text a CLI matches on — it must say WHEN")
+
+
[email protected]("d", BUNDLES, ids=IDS)
+def test_no_angle_bracket_anywhere_in_the_frontmatter(d: Path):
+ """The spec's prompt-injection rule: frontmatter is pasted into a system prompt."""
+ text = (d / "SKILL.md").read_text(encoding="utf-8")
+ fm_text = re.match(r"\A---[ \t]*\r?\n(.*?)\r?\n---", text, re.S).group(1)
+ assert "<" not in fm_text and ">" not in fm_text
+
+
[email protected]("d", BUNDLES, ids=IDS)
+def test_only_spec_keys_at_the_top_level_and_metadata_is_a_string_map(d: Path):
+ fm, _ = _raw(d)
+ assert set(fm) <= SPEC_KEYS, f"{d.name}: {sorted(set(fm) - SPEC_KEYS)} belong under `metadata`"
+ meta = fm.get("metadata") or {}
+ assert isinstance(meta, dict)
+ for k, v in meta.items():
+ assert isinstance(k, str) and isinstance(v, (str, int, float, bool)), (
+ f"{d.name}: metadata.{k} must be a scalar (the spec allows string→string)")
+ if "compatibility" in fm:
+ assert len(str(fm["compatibility"])) <= COMPATIBILITY_MAX
+
+
[email protected]("d", BUNDLES, ids=IDS)
+def test_the_body_stays_under_the_spec_line_budget(d: Path):
+ _, body = _raw(d)
+ assert len(body.splitlines()) <= BODY_MAX_LINES
+
+
[email protected]("d", BUNDLES, ids=IDS)
+def test_resources_are_one_level_deep_in_the_directories_the_spec_names(d: Path):
+ for p in d.iterdir():
+ if p.is_file():
+ assert p.name == "SKILL.md", f"{d.name}: stray file {p.name}; depth belongs in references/"
+ continue
+ assert p.name in OPTIONAL_DIRS, f"{d.name}: {p.name}/ is not one of {sorted(OPTIONAL_DIRS)}"
+ for child in p.rglob("*"):
+ assert child.is_file(), f"{d.name}: {child.relative_to(d)} is a second level of nesting"
+
+
[email protected]("d", BUNDLES, ids=IDS)
+def test_every_relative_link_in_the_body_resolves_inside_the_bundle(d: Path):
+ """Progressive disclosure only works if the pointer points at something."""
+ _, body = _raw(d)
+ for target in re.findall(r"\]\((?!https?:|#)([^)]+)\)", body):
+ rel = target.split("#", 1)[0].strip()
+ if not rel:
+ continue
+ assert not rel.startswith("/") and ".." not in Path(rel).parts, f"{d.name}: {rel} escapes the bundle"
+ assert (d / rel).exists(), f"{d.name}: links to {rel}, which does not exist"
+
+
[email protected](REF_VALIDATOR is None,
+ reason="the reference validator is not installed (pip install skills-ref)")
[email protected]("d", BUNDLES, ids=IDS)
+def test_the_reference_implementation_validates_the_bundle(d: Path):
+ p = subprocess.run([REF_VALIDATOR, "validate", str(d)], capture_output=True, text=True, check=False)
+ assert p.returncode == 0, p.stdout + p.stderr
+
+
[email protected](REF_VALIDATOR is None, reason="the reference validator is not installed")
[email protected]("d", BUNDLES, ids=IDS)
+def test_our_loader_and_the_reference_implementation_read_the_same_fields(d: Path):
+ """A disagreement here is the failure mode that matters: we index one description,
+ the CLI indexes another, and the read rate measures a skill nobody was offered."""
+ from codeverse.skills.loader import parse_skill
+
+ p = subprocess.run([REF_VALIDATOR, "read-properties", str(d)],
+ capture_output=True, text=True, check=False)
+ assert p.returncode == 0, p.stdout + p.stderr
+ theirs = json.loads(p.stdout)
+ ours = parse_skill(d)
+ assert theirs["name"] == ours.name
+ assert theirs["description"] == ours.description
+ assert dict(theirs.get("metadata") or {}) == dict(ours.metadata)