Official Codebase of 3DCodeVerse
Python
tests: the harness suite passes without eval/ — a link or battery id into an absent eval/ skips with its name
gaoypeng4f2b7bca2b22
diff --git a/harness/tests/install/test_docs.py b/harness/tests/install/test_docs.pyindex 894e6116..f1dc452a 100644--- a/harness/tests/install/test_docs.py+++ b/harness/tests/install/test_docs.py@@ -13,6 +13,7 @@ import pytestHARNESS = Path(__file__).resolve().parents[2]REPO = HARNESS.parent+EVAL = REPO / "eval"INSTALL = HARNESS / "docs" / "INSTALL.md"SETUP = HARNESS / "setup.sh"@@ -93,6 +94,11 @@ def existing_docs() -> list[Path]:return [p for p in OWNED_DOCS if p.is_file()]+def into_absent_eval(path: Path) -> bool:+ """A path into eval/ on a checkout without it: the harness suite passes without eval/ (D77)."""+ return not EVAL.is_dir() and path.resolve().is_relative_to(EVAL)++# --------------------------------------------------------------------------- testsdef test_install_guide_exists() -> None:assert INSTALL.is_file(), f"{INSTALL} is missing"@@ -117,6 +123,7 @@ def test_owned_docs_have_valid_commands_links_and_paths(doc: Path) -> None:headings = {slugify(ln) for ln in text.splitlines() if ln.startswith("#")}missing: list[str] = []+ into_eval: list[str] = []for target in _MD_LINK.findall(text):if target.startswith(("http://", "https://", "mailto:", "<")):continue@@ -126,16 +133,19 @@ def test_owned_docs_have_valid_commands_links_and_paths(doc: Path) -> None:continuepath, _, anchor = target.partition("#")if path and not (doc.parent / path).exists():- missing.append(f"link {target}")+ (into_eval if into_absent_eval(doc.parent / path) else missing).append(f"link {target}")assert not missing, f"{doc}: unresolved links: {missing}"missing_paths = []for ref in sorted(set(_BACKTICK_MD.findall(text))):if ref.startswith("~"):continue- if not any((base / ref).exists() for base in (doc.parent, HARNESS, REPO)):- missing_paths.append(ref)+ bases = (doc.parent, HARNESS, REPO)+ if not any((base / ref).exists() for base in bases):+ (into_eval if any(into_absent_eval(base / ref) for base in bases) else missing_paths).append(ref)assert not missing_paths, f"{doc}: unresolved backticked paths: {missing_paths}"+ if into_eval:+ pytest.skip(f"no eval/ beside the harness: cannot resolve {into_eval}")def test_setup_script_is_executable_parses_and_has_help() -> None:diff --git a/harness/tests/skills/test_freshness.py b/harness/tests/skills/test_freshness.pyindex 9a481327..e322327b 100644--- a/harness/tests/skills/test_freshness.py+++ b/harness/tests/skills/test_freshness.py@@ -20,6 +20,7 @@ SKILLS = list(iter_skills()) if BUNDLES else []pytestmark = pytest.mark.skipif(not BUNDLES, reason=f"no bundles in {skills_dir()} yet")HARNESS = Path(__file__).resolve().parents[2]+EVAL_BENCH = HARNESS.parent / "eval" / "bench" # absent on a harness-only checkout (D77)#: every markdown file a bundle ships — SKILL.md and its references are equally quotableDOCS: list[Path] = sorted(p for d in BUNDLES for p in d.rglob("*.md"))@@ -87,7 +88,7 @@ def _live_vocabulary() -> set[str]:"""words: set[str] = set()for root, exts in ((HARNESS / "codeverse3d", {".py", ".md", ".j2", ".yaml", ".yml", ".toml"}),- (HARNESS.parent / "eval" / "bench", {".py", ".yaml", ".yml"}), # battery + prompt ids a bundle cites+ (EVAL_BENCH, {".py", ".yaml", ".yml"}), # battery + prompt ids a bundle cites(HARNESS / "runtime_js", {".js", ".mjs", ".glsl", ".json"})):if not root.is_dir():continue@@ -98,7 +99,7 @@ def _live_vocabulary() -> set[str]:or p.parent.parent.name.startswith("c3d-")):continuewords |= set(re.findall(r"[A-Za-z_][A-Za-z0-9_]*", p.read_text(errors="ignore")))- out = HARNESS.parent / "eval" / "bench" / "out"+ out = EVAL_BENCH / "out"if out.is_dir():words |= {d.name for d in out.iterdir() if d.is_dir()}return words@@ -155,6 +156,7 @@ def test_every_bundle_document_only_cites_live_vocabulary():d: "\n".join(_FENCE.findall("\n".join(p.read_text() for p in d.rglob("*.md"))))for d in BUNDLES}+ unverifiable: dict[str, list[str]] = {} # without eval/bench a battery id reads as stalefor doc in DOCS:label = doc.relative_to(skills_dir())raw = doc.read_text()@@ -163,7 +165,10 @@ def test_every_bundle_document_only_cites_live_vocabulary():unknown = sorted({t for t in named if _HARNESS_SHAPED.match(t)} - LIVE_VOCABULARY - FOREIGN_CALLS)- assert not unknown, f"{label} quotes names absent from the harness: {unknown}"+ if unknown and not EVAL_BENCH.is_dir():+ unverifiable[str(label)] = unknown+ else:+ assert not unknown, f"{label} quotes names absent from the harness: {unknown}"called = {match.group(1)@@ -189,6 +194,8 @@ def test_every_bundle_document_only_cites_live_vocabulary():siblings = set(re.findall(r"\bc3d-[a-z0-9-]+\b", raw))assert siblings <= have, f"{label} points at missing skills: {sorted(siblings - have)}"+ if unverifiable:+ pytest.skip(f"no eval/bench beside the harness to tell battery ids from stale names: {unverifiable}")def _languages_of(skill_name: str) -> set[str]: