Skip to content

Official Codebase of 3DCodeVerse

Python

tests: the harness suite passes without eval/ — a link or battery id into an absent eval/ skips with its name

gaoypeng4f2b7bca2b22
diff --git a/harness/tests/install/test_docs.py b/harness/tests/install/test_docs.py
index 894e6116..f1dc452a 100644
--- a/harness/tests/install/test_docs.py
+++ b/harness/tests/install/test_docs.py
@@ -13,6 +13,7 @@ import pytest
HARNESS = Path(__file__).resolve().parents[2]
REPO = HARNESS.parent
+EVAL = REPO / "eval"
INSTALL = HARNESS / "docs" / "INSTALL.md"
SETUP = HARNESS / "setup.sh"
@@ -93,6 +94,11 @@ def existing_docs() -> list[Path]:
return [p for p in OWNED_DOCS if p.is_file()]
+def into_absent_eval(path: Path) -> bool:
+ """A path into eval/ on a checkout without it: the harness suite passes without eval/ (D77)."""
+ return not EVAL.is_dir() and path.resolve().is_relative_to(EVAL)
+
+
# --------------------------------------------------------------------------- tests
def test_install_guide_exists() -> None:
assert INSTALL.is_file(), f"{INSTALL} is missing"
@@ -117,6 +123,7 @@ def test_owned_docs_have_valid_commands_links_and_paths(doc: Path) -> None:
headings = {slugify(ln) for ln in text.splitlines() if ln.startswith("#")}
missing: list[str] = []
+ into_eval: list[str] = []
for target in _MD_LINK.findall(text):
if target.startswith(("http://", "https://", "mailto:", "<")):
continue
@@ -126,16 +133,19 @@ def test_owned_docs_have_valid_commands_links_and_paths(doc: Path) -> None:
continue
path, _, anchor = target.partition("#")
if path and not (doc.parent / path).exists():
- missing.append(f"link {target}")
+ (into_eval if into_absent_eval(doc.parent / path) else missing).append(f"link {target}")
assert not missing, f"{doc}: unresolved links: {missing}"
missing_paths = []
for ref in sorted(set(_BACKTICK_MD.findall(text))):
if ref.startswith("~"):
continue
- if not any((base / ref).exists() for base in (doc.parent, HARNESS, REPO)):
- missing_paths.append(ref)
+ bases = (doc.parent, HARNESS, REPO)
+ if not any((base / ref).exists() for base in bases):
+ (into_eval if any(into_absent_eval(base / ref) for base in bases) else missing_paths).append(ref)
assert not missing_paths, f"{doc}: unresolved backticked paths: {missing_paths}"
+ if into_eval:
+ pytest.skip(f"no eval/ beside the harness: cannot resolve {into_eval}")
def test_setup_script_is_executable_parses_and_has_help() -> None:
diff --git a/harness/tests/skills/test_freshness.py b/harness/tests/skills/test_freshness.py
index 9a481327..e322327b 100644
--- a/harness/tests/skills/test_freshness.py
+++ b/harness/tests/skills/test_freshness.py
@@ -20,6 +20,7 @@ SKILLS = list(iter_skills()) if BUNDLES else []
pytestmark = pytest.mark.skipif(not BUNDLES, reason=f"no bundles in {skills_dir()} yet")
HARNESS = Path(__file__).resolve().parents[2]
+EVAL_BENCH = HARNESS.parent / "eval" / "bench" # absent on a harness-only checkout (D77)
#: every markdown file a bundle ships — SKILL.md and its references are equally quotable
DOCS: list[Path] = sorted(p for d in BUNDLES for p in d.rglob("*.md"))
@@ -87,7 +88,7 @@ def _live_vocabulary() -> set[str]:
"""
words: set[str] = set()
for root, exts in ((HARNESS / "codeverse3d", {".py", ".md", ".j2", ".yaml", ".yml", ".toml"}),
- (HARNESS.parent / "eval" / "bench", {".py", ".yaml", ".yml"}), # battery + prompt ids a bundle cites
+ (EVAL_BENCH, {".py", ".yaml", ".yml"}), # battery + prompt ids a bundle cites
(HARNESS / "runtime_js", {".js", ".mjs", ".glsl", ".json"})):
if not root.is_dir():
continue
@@ -98,7 +99,7 @@ def _live_vocabulary() -> set[str]:
or p.parent.parent.name.startswith("c3d-")):
continue
words |= set(re.findall(r"[A-Za-z_][A-Za-z0-9_]*", p.read_text(errors="ignore")))
- out = HARNESS.parent / "eval" / "bench" / "out"
+ out = EVAL_BENCH / "out"
if out.is_dir():
words |= {d.name for d in out.iterdir() if d.is_dir()}
return words
@@ -155,6 +156,7 @@ def test_every_bundle_document_only_cites_live_vocabulary():
d: "\n".join(_FENCE.findall("\n".join(p.read_text() for p in d.rglob("*.md"))))
for d in BUNDLES
}
+ unverifiable: dict[str, list[str]] = {} # without eval/bench a battery id reads as stale
for doc in DOCS:
label = doc.relative_to(skills_dir())
raw = doc.read_text()
@@ -163,7 +165,10 @@ def test_every_bundle_document_only_cites_live_vocabulary():
unknown = sorted(
{t for t in named if _HARNESS_SHAPED.match(t)} - LIVE_VOCABULARY - FOREIGN_CALLS
)
- assert not unknown, f"{label} quotes names absent from the harness: {unknown}"
+ if unknown and not EVAL_BENCH.is_dir():
+ unverifiable[str(label)] = unknown
+ else:
+ assert not unknown, f"{label} quotes names absent from the harness: {unknown}"
called = {
match.group(1)
@@ -189,6 +194,8 @@ def test_every_bundle_document_only_cites_live_vocabulary():
siblings = set(re.findall(r"\bc3d-[a-z0-9-]+\b", raw))
assert siblings <= have, f"{label} points at missing skills: {sorted(siblings - have)}"
+ if unverifiable:
+ pytest.skip(f"no eval/bench beside the harness to tell battery ids from stale names: {unverifiable}")
def _languages_of(skill_name: str) -> set[str]: