Official Codebase of 3DCodeVerse
Python
Project checks: a project's deterministic checks join every round's gates as project:<name>
wang Qin11e9ab33febe
diff --git a/eval/bench/run_bench.py b/eval/bench/run_bench.pyindex 2e72271f..88e1849b 100644--- a/eval/bench/run_bench.py+++ b/eval/bench/run_bench.py@@ -53,7 +53,13 @@ from codeverse3d.addons import select # noqa: E402from codeverse3d.config import get_settings # noqa: E402from codeverse3d.contracts.common import TRACK_INFO, Backends, Budget, Language, Track # noqa: E402from codeverse3d.contracts.run import RunRecord # noqa: E402-from codeverse3d.contracts.spec import Constraints, ReferenceImage, RunOptions, Spec # noqa: E402+from codeverse3d.contracts.spec import ( # noqa: E402+ Constraints,+ ProjectCheck,+ ReferenceImage,+ RunOptions,+ Spec,+)from codeverse3d.cost import run_ledger # noqa: E402from codeverse3d.proc import exclusive, install_stop_handlers # noqa: E402from codeverse3d.record.record import load_record # noqa: E402@@ -79,6 +85,9 @@ class BenchPrompt(BaseModel):references: list[str] = Field(default_factory=list,description="reference image paths (relative to the battery file); bench/refs/<id>/*.png|jpg are added automatically")+ # left out of a dumped battery while empty, so every existing battery serialises as before+ checks: list[ProjectCheck] = Field(default_factory=list, exclude_if=lambda v: not v,+ description="project checks run every round (Spec.checks)")class Battery(BaseModel):@@ -191,6 +200,7 @@ def build_spec(budget=Budget(max_rounds=rounds, max_minutes=max_minutes),backends=backends, tags=[tag0, battery.name, item.tier, item.category, *extra_tags, *item.tags],options=RunOptions(variant=variant),+ checks=list(item.checks),)diff --git a/eval/tests/test_bench_shared.py b/eval/tests/test_bench_shared.pyindex 6682ed4d..8a537d57 100644--- a/eval/tests/test_bench_shared.py+++ b/eval/tests/test_bench_shared.py@@ -69,3 +69,24 @@ def test_a_run_that_scored_nothing_says_so_in_its_row(tmp_path: Path) -> None:rec.rounds = [judged, rounds[1]]row = result_from_record(item, rec, ws)assert row.errors == "" and (row.score_picked, row.picked_round) == (0.6, 0) # one verdict is enough+++def test_a_battery_prompt_carries_checks_into_the_spec(tmp_path):+ """A project registers checks per prompt: build_spec must hand them to the run (it builds the Spec from the+ battery even when the workspace already holds a spec.json), and a prompt without them must serialise exactly+ as before."""+ import yaml++ from bench.run_bench import Battery, build_spec+ from codeverse3d.contracts.common import Backends++ path = tmp_path / "b.yaml"+ path.write_text(yaml.safe_dump({"name": "b", "track": "static_object", "language": "threejs", "prompts": [+ {"id": "house", "prompt": "a house", "tier": "hard",+ "checks": [{"name": "survey", "entry": "pkg.mod:check", "args": {"id": "house"}}]},+ {"id": "plain", "prompt": "a chair", "tier": "easy"}]}))+ b = Battery.load(path)+ spec = build_spec(b, b.prompts[0], backends=Backends(), rounds=1, max_minutes=1, tag0="t")+ assert spec.checks[0].args == {"id": "house"}+ plain = build_spec(b, b.prompts[1], backends=Backends(), rounds=1, max_minutes=1, tag0="t").model_dump(mode="json")+ assert "checks" not in plaindiff --git a/harness/codeverse3d/contracts/spec.py b/harness/codeverse3d/contracts/spec.pyindex 612e4b96..bb1168fa 100644--- a/harness/codeverse3d/contracts/spec.py+++ b/harness/codeverse3d/contracts/spec.py@@ -3,6 +3,7 @@from __future__ import annotationsfrom datetime import UTC, datetime+from typing import Anyfrom pydantic import BaseModel, Field, model_validator@@ -27,6 +28,18 @@ class Constraints(BaseModel):must_not: list[str] = Field(default_factory=list)+class ProjectCheck(BaseModel):+ """A deterministic check the PROJECT registers (``Spec.checks``): ``entry`` names a function+ (``package.module:function``, importable where the harness runs) that measures each round's GLB against+ what the project knows — survey geometry, catalogue dimensions — and returns ``GateReport``s. They take part+ exactly like the built-in gates under the name ``project:<name>``: ERRORs become refine tasks, the judge reads+ them, and a rubric ``CapRule(when="gate", gate="project:*")`` caps on them (``tracks/project_checks.py``)."""++ name: str = Field(pattern=r"^[a-z0-9][a-z0-9_]*$", description="gate suffix: reports are named project:<name>")+ entry: str = Field(pattern=r"^[A-Za-z_][\w.]*:[A-Za-z_]\w*$", description="'package.module:function'")+ args: dict[str, Any] = Field(default_factory=dict, description="keyword data passed to the check as args=")++class RunOptions(BaseModel):"""Run-shape options frozen on the spec (distinct from Budget and Backends).@@ -47,6 +60,9 @@ class Spec(BaseModel):prompt: strreferences: list[ReferenceImage] = Field(default_factory=list)constraints: Constraints = Field(default_factory=Constraints)+ # left out of spec.json while empty, so every spec without one serialises exactly as before+ checks: list[ProjectCheck] = Field(default_factory=list, exclude_if=lambda v: not v,+ description="project checks run on every round's GLB (ProjectCheck)")budget: Budget = Field(default_factory=Budget)backends: Backends = Field(default_factory=Backends)options: RunOptions = Field(default_factory=RunOptions)diff --git a/harness/codeverse3d/judges/rubrics/reference.yaml b/harness/codeverse3d/judges/rubrics/reference.yamlindex a42c5592..a1a25472 100644--- a/harness/codeverse3d/judges/rubrics/reference.yaml+++ b/harness/codeverse3d/judges/rubrics/reference.yaml@@ -119,6 +119,13 @@ caps:kinds: [penetrat, intersect, overlap, collision]cap: 0.7measures: [interpenetration]+ # a deterministic check the project registered (Spec.checks, tracks/project_checks.py) measured an error+ - id: project_check+ when: gate+ gate: "project:*"+ severity: error+ cap: 0.6+ note: the project's own measurement of the object failed- id: missing_must_acceptancewhen: acceptancecap: 0.6diff --git a/harness/codeverse3d/judges/rubrics/static_object.yaml b/harness/codeverse3d/judges/rubrics/static_object.yamlindex e7ea9e4b..59df000f 100644--- a/harness/codeverse3d/judges/rubrics/static_object.yaml+++ b/harness/codeverse3d/judges/rubrics/static_object.yaml@@ -176,6 +176,13 @@ caps:severity: errorcap: 0.75note: measured bbox/part contract broken beyond tolerance+ # a deterministic check the project registered (Spec.checks, tracks/project_checks.py) measured an error+ - id: project_check+ when: gate+ gate: "project:*"+ severity: error+ cap: 0.6+ note: the project's own measurement of the object failed- id: missing_must_acceptancewhen: acceptancecap: 0.6diff --git a/harness/codeverse3d/tracks/project_checks.py b/harness/codeverse3d/tracks/project_checks.pynew file mode 100644index 00000000..ff422a46--- /dev/null+++ b/harness/codeverse3d/tracks/project_checks.py@@ -0,0 +1,77 @@+"""Project checks (``Spec.checks``): deterministic checks a project registers by import path, run on every round.++The harness gates what it can know about any object — the build, the plan contract, connectivity, budgets. A+project often knows more: the true geometry of the thing (a surveyed building, a catalogue part), definitions+of its own (a building's eaves line) and thresholds. A registered check's reports join the round's gates under+``project:<name>``: an ERROR becomes a refine task with its ``fix_hint``, the judge reads it in the gates section,+and a rubric ``CapRule(when="gate", gate="project:*")`` caps the score.++A check is ``fn(*, glb, workspace, spec, measurement, round_index, args) -> GateReport | list[GateReport]``. It+must be deterministic and must not call a model (Law 3). A check that raises costs the round nothing but a WARN+saying so: a project's bug must not fail or cap the run.+"""++from __future__ import annotations++import importlib+import logging+import time+from collections.abc import Sequence+from pathlib import Path+from typing import Any, Protocol++from codeverse3d.contracts.artifacts import GateFinding, GateReport, Measurement, Severity+from codeverse3d.contracts.spec import ProjectCheck, Spec++log = logging.getLogger(__name__)++PREFIX = "project:"+++class CheckFn(Protocol):+ def __call__(self, *, glb: Path, workspace: Path, spec: Spec, measurement: Measurement | None,+ round_index: int, args: dict[str, Any]) -> GateReport | Sequence[GateReport]: ...+++def load_check(entry: str) -> CheckFn:+ """The function ``entry`` (``package.module:function``) names."""+ module, _, attr = entry.partition(":")+ fn = getattr(importlib.import_module(module), attr)+ if not callable(fn):+ raise TypeError(f"{entry} is not callable")+ return fn+++def _named(report: GateReport, check: ProjectCheck) -> GateReport:+ """``report`` under ``project:<name>`` (a check returning several reports keeps them apart as+ ``project:<name>.<its gate>``), its findings renamed with it, so caps and refine keys see one namespace."""+ own = report.gate or ""+ if own.startswith(PREFIX):+ gate = own+ elif own in ("", check.name):+ gate = PREFIX + check.name+ else:+ gate = f"{PREFIX}{check.name}.{own}"+ findings = [f.model_copy(update={"gate": gate}) for f in report.findings]+ return report.model_copy(update={"gate": gate, "findings": findings})+++def run_project_checks(spec: Spec, workspace: Path, glb: Path, measurement: Measurement | None,+ round_index: int) -> list[GateReport]:+ """Every check of ``spec`` on this round's GLB, in the spec's order."""+ out: list[GateReport] = []+ for check in spec.checks:+ t0 = time.time()+ try:+ got = load_check(check.entry)(glb=glb, workspace=workspace, spec=spec, measurement=measurement,+ round_index=round_index, args=dict(check.args))+ reports = [got] if isinstance(got, GateReport) else list(got)+ out += [_named(r, check) for r in reports]+ except Exception as e: # a project's bug must not fail or cap the run+ log.warning("project check %s failed: %s", check.entry, e)+ gate = PREFIX + check.name+ out.append(GateReport.of(gate, [GateFinding(+ gate=gate, severity=Severity.WARN,+ message=f"project check {check.entry} raised {type(e).__name__}: {str(e)[:300]}")],+ duration_ms=int((time.time() - t0) * 1000)))+ return outdiff --git a/harness/codeverse3d/tracks/steps.py b/harness/codeverse3d/tracks/steps.pyindex 992233f1..426f3faa 100644--- a/harness/codeverse3d/tracks/steps.py+++ b/harness/codeverse3d/tracks/steps.py@@ -28,6 +28,7 @@ from codeverse3d.spatial.render import RenderErrorfrom codeverse3d.tracks import skills_hookfrom codeverse3d.tracks.common import RunContext, generate_forfrom codeverse3d.tracks.generation import GenerationResult, GenerationTask+from codeverse3d.tracks.project_checks import run_project_checksfrom codeverse3d.tracks.repair import RepairOutcome, build_with_repairfrom codeverse3d.workspace import Workspace@@ -189,6 +190,9 @@ def _run_round(with timed("gates", steps, round_index=index):rec.measurement = pipeline.measure(ctx, outcome.build)gates.extend(pipeline.gates(ctx, index, outcome.build, rec.measurement))+ if ctx.spec.checks and outcome.build.glb_path: # the project's own checks (Spec.checks)+ gates.extend(run_project_checks(ctx.spec, ctx.ws.root, Path(outcome.build.glb_path),+ rec.measurement, index))try:with timed("render", steps, round_index=index):rec.renders = (render or pipeline.render)(ctx, index, outcome.build, rec.measurement)diff --git a/harness/docs/ARCHITECTURE.md b/harness/docs/ARCHITECTURE.mdindex d1b0e236..e1ed5e7c 100644--- a/harness/docs/ARCHITECTURE.md+++ b/harness/docs/ARCHITECTURE.md@@ -157,7 +157,8 @@ codeverse3d/single-shot asset generation), zone_layout.py (per-zone structured layout calls +deterministic validator), graphics.py (the whole graphics track), plan_features.py(eval-only: which C3D_* names are live, and pin_plan_blockers() deciding when two arms may- share one plan), depth.py, skills_hook.py (the round's view of codeverse3d/skills: attach+ share one plan), depth.py, project_checks.py (Spec.checks: the project's deterministic checks, run on+ every built round and joined to its gates as project:<name>), skills_hook.py (the round's view of codeverse3d/skills: attachbefore generating, probe reads after — a no-op under C3D_SKILLS=0)record/ what every run WRITES: record.py (finalize_record, load_record, iter_runs,effective_judgment), deliverable.py (every round's kept build under artifacts/rNN/, anddiff --git a/harness/tests/orchestrator_tracks/test_project_checks.py b/harness/tests/orchestrator_tracks/test_project_checks.pynew file mode 100644index 00000000..d76ddc06--- /dev/null+++ b/harness/tests/orchestrator_tracks/test_project_checks.py@@ -0,0 +1,78 @@+"""Spec.checks: a project's deterministic checks join every round's gates as project:<name>."""++from __future__ import annotations++from pathlib import Path+from typing import Any++import pytest++from codeverse3d.contracts.artifacts import GateFinding, GateReport, Severity+from codeverse3d.contracts.common import Language, Track+from codeverse3d.contracts.spec import ProjectCheck, Spec+from codeverse3d.tracks.project_checks import run_project_checks++HERE = "tests.orchestrator_tracks.test_project_checks"+++def roof_check(*, glb: Path, workspace: Path, spec: Spec, measurement: Any, round_index: int,+ args: dict[str, Any]) -> GateReport:+ within = args["within"]+ sev = Severity.ERROR if within < 0.85 else Severity.INFO+ return GateReport.of("", [GateFinding(gate="", severity=sev, target="Roof",+ message=f"{within:.0%} of the roof within 0.5 m of the survey (round {round_index})",+ fix_hint="build the roof planes from the survey", data={"kind": "roof_shape"})])+++def two_reports(**_: Any) -> list[GateReport]:+ return [GateReport.of("eaves", []), GateReport.of("project:other", [])]+++def broken(**_: Any) -> GateReport:+ raise RuntimeError("reference mesh missing")+++def _spec(*checks: ProjectCheck) -> Spec:+ return Spec(id="x", track=Track.STATIC_OBJECT, language=Language.THREEJS, prompt="a house", checks=list(checks))+++def test_checks_are_named_and_imported_by_path() -> None:+ with pytest.raises(ValueError):+ ProjectCheck(name="Survey Check", entry=f"{HERE}:roof_check")+ with pytest.raises(ValueError):+ ProjectCheck(name="survey", entry="no_function_here")+ assert "checks" not in _spec().model_dump(mode="json") # a spec without checks serialises as before+++def test_a_check_joins_the_round_as_project_gate(tmp_path: Path) -> None:+ spec = _spec(ProjectCheck(name="survey", entry=f"{HERE}:roof_check", args={"within": 0.59}))+ [rep] = run_project_checks(spec, tmp_path, tmp_path / "object.glb", None, 2)+ assert rep.gate == "project:survey" and not rep.passed+ err = rep.errors[0]+ assert err.gate == "project:survey" and err.data["kind"] == "roof_shape" and "(round 2)" in err.message++ spec = _spec(ProjectCheck(name="survey", entry=f"{HERE}:two_reports"))+ assert [r.gate for r in run_project_checks(spec, tmp_path, tmp_path / "o.glb", None, 0)] == [+ "project:survey.eaves", "project:other"]+++def test_a_broken_check_warns_and_never_fails_the_round(tmp_path: Path) -> None:+ spec = _spec(ProjectCheck(name="survey", entry=f"{HERE}:broken"),+ ProjectCheck(name="missing", entry="no_such_module_here:check"))+ reps = run_project_checks(spec, tmp_path, tmp_path / "o.glb", None, 0)+ assert [r.gate for r in reps] == ["project:survey", "project:missing"]+ assert all(r.passed for r in reps)+ assert "RuntimeError: reference mesh missing" in reps[0].findings[0].message+ assert reps[1].findings[0].severity is Severity.WARN++[email protected]("rubric", ["reference", "static_object"])+def test_a_project_error_caps_the_score(tmp_path: Path, rubric: str) -> None:+ from codeverse3d.judges.rubrics import apply_caps, load_rubric++ spec = _spec(ProjectCheck(name="survey", entry=f"{HERE}:roof_check", args={"within": 0.59}))+ gates = run_project_checks(spec, tmp_path, tmp_path / "o.glb", None, 0)+ assert apply_caps(load_rubric(rubric), 0.9, gates, {}).overall == pytest.approx(0.6)+ ok = _spec(ProjectCheck(name="survey", entry=f"{HERE}:roof_check", args={"within": 0.97}))+ assert apply_caps(load_rubric(rubric), 0.9, run_project_checks(ok, tmp_path, tmp_path / "o.glb", None, 0),+ {}).overall == pytest.approx(0.9)