Skip to content

Official Codebase of 3DCodeVerse

Python

Project checks: a project's deterministic checks join every round's gates as project:<name>

wang Qin11e9ab33febe
diff --git a/eval/bench/run_bench.py b/eval/bench/run_bench.py
index 2e72271f..88e1849b 100644
--- a/eval/bench/run_bench.py
+++ b/eval/bench/run_bench.py
@@ -53,7 +53,13 @@ from codeverse3d.addons import select # noqa: E402
from codeverse3d.config import get_settings # noqa: E402
from codeverse3d.contracts.common import TRACK_INFO, Backends, Budget, Language, Track # noqa: E402
from codeverse3d.contracts.run import RunRecord # noqa: E402
-from codeverse3d.contracts.spec import Constraints, ReferenceImage, RunOptions, Spec # noqa: E402
+from codeverse3d.contracts.spec import ( # noqa: E402
+ Constraints,
+ ProjectCheck,
+ ReferenceImage,
+ RunOptions,
+ Spec,
+)
from codeverse3d.cost import run_ledger # noqa: E402
from codeverse3d.proc import exclusive, install_stop_handlers # noqa: E402
from codeverse3d.record.record import load_record # noqa: E402
@@ -79,6 +85,9 @@ class BenchPrompt(BaseModel):
references: list[str] = Field(
default_factory=list,
description="reference image paths (relative to the battery file); bench/refs/<id>/*.png|jpg are added automatically")
+ # left out of a dumped battery while empty, so every existing battery serialises as before
+ checks: list[ProjectCheck] = Field(default_factory=list, exclude_if=lambda v: not v,
+ description="project checks run every round (Spec.checks)")
class Battery(BaseModel):
@@ -191,6 +200,7 @@ def build_spec(
budget=Budget(max_rounds=rounds, max_minutes=max_minutes),
backends=backends, tags=[tag0, battery.name, item.tier, item.category, *extra_tags, *item.tags],
options=RunOptions(variant=variant),
+ checks=list(item.checks),
)
diff --git a/eval/tests/test_bench_shared.py b/eval/tests/test_bench_shared.py
index 6682ed4d..8a537d57 100644
--- a/eval/tests/test_bench_shared.py
+++ b/eval/tests/test_bench_shared.py
@@ -69,3 +69,24 @@ def test_a_run_that_scored_nothing_says_so_in_its_row(tmp_path: Path) -> None:
rec.rounds = [judged, rounds[1]]
row = result_from_record(item, rec, ws)
assert row.errors == "" and (row.score_picked, row.picked_round) == (0.6, 0) # one verdict is enough
+
+
+def test_a_battery_prompt_carries_checks_into_the_spec(tmp_path):
+ """A project registers checks per prompt: build_spec must hand them to the run (it builds the Spec from the
+ battery even when the workspace already holds a spec.json), and a prompt without them must serialise exactly
+ as before."""
+ import yaml
+
+ from bench.run_bench import Battery, build_spec
+ from codeverse3d.contracts.common import Backends
+
+ path = tmp_path / "b.yaml"
+ path.write_text(yaml.safe_dump({"name": "b", "track": "static_object", "language": "threejs", "prompts": [
+ {"id": "house", "prompt": "a house", "tier": "hard",
+ "checks": [{"name": "survey", "entry": "pkg.mod:check", "args": {"id": "house"}}]},
+ {"id": "plain", "prompt": "a chair", "tier": "easy"}]}))
+ b = Battery.load(path)
+ spec = build_spec(b, b.prompts[0], backends=Backends(), rounds=1, max_minutes=1, tag0="t")
+ assert spec.checks[0].args == {"id": "house"}
+ plain = build_spec(b, b.prompts[1], backends=Backends(), rounds=1, max_minutes=1, tag0="t").model_dump(mode="json")
+ assert "checks" not in plain
diff --git a/harness/codeverse3d/contracts/spec.py b/harness/codeverse3d/contracts/spec.py
index 612e4b96..bb1168fa 100644
--- a/harness/codeverse3d/contracts/spec.py
+++ b/harness/codeverse3d/contracts/spec.py
@@ -3,6 +3,7 @@
from __future__ import annotations
from datetime import UTC, datetime
+from typing import Any
from pydantic import BaseModel, Field, model_validator
@@ -27,6 +28,18 @@ class Constraints(BaseModel):
must_not: list[str] = Field(default_factory=list)
+class ProjectCheck(BaseModel):
+ """A deterministic check the PROJECT registers (``Spec.checks``): ``entry`` names a function
+ (``package.module:function``, importable where the harness runs) that measures each round's GLB against
+ what the project knows — survey geometry, catalogue dimensions — and returns ``GateReport``s. They take part
+ exactly like the built-in gates under the name ``project:<name>``: ERRORs become refine tasks, the judge reads
+ them, and a rubric ``CapRule(when="gate", gate="project:*")`` caps on them (``tracks/project_checks.py``)."""
+
+ name: str = Field(pattern=r"^[a-z0-9][a-z0-9_]*$", description="gate suffix: reports are named project:<name>")
+ entry: str = Field(pattern=r"^[A-Za-z_][\w.]*:[A-Za-z_]\w*$", description="'package.module:function'")
+ args: dict[str, Any] = Field(default_factory=dict, description="keyword data passed to the check as args=")
+
+
class RunOptions(BaseModel):
"""Run-shape options frozen on the spec (distinct from Budget and Backends).
@@ -47,6 +60,9 @@ class Spec(BaseModel):
prompt: str
references: list[ReferenceImage] = Field(default_factory=list)
constraints: Constraints = Field(default_factory=Constraints)
+ # left out of spec.json while empty, so every spec without one serialises exactly as before
+ checks: list[ProjectCheck] = Field(default_factory=list, exclude_if=lambda v: not v,
+ description="project checks run on every round's GLB (ProjectCheck)")
budget: Budget = Field(default_factory=Budget)
backends: Backends = Field(default_factory=Backends)
options: RunOptions = Field(default_factory=RunOptions)
diff --git a/harness/codeverse3d/judges/rubrics/reference.yaml b/harness/codeverse3d/judges/rubrics/reference.yaml
index a42c5592..a1a25472 100644
--- a/harness/codeverse3d/judges/rubrics/reference.yaml
+++ b/harness/codeverse3d/judges/rubrics/reference.yaml
@@ -119,6 +119,13 @@ caps:
kinds: [penetrat, intersect, overlap, collision]
cap: 0.7
measures: [interpenetration]
+ # a deterministic check the project registered (Spec.checks, tracks/project_checks.py) measured an error
+ - id: project_check
+ when: gate
+ gate: "project:*"
+ severity: error
+ cap: 0.6
+ note: the project's own measurement of the object failed
- id: missing_must_acceptance
when: acceptance
cap: 0.6
diff --git a/harness/codeverse3d/judges/rubrics/static_object.yaml b/harness/codeverse3d/judges/rubrics/static_object.yaml
index e7ea9e4b..59df000f 100644
--- a/harness/codeverse3d/judges/rubrics/static_object.yaml
+++ b/harness/codeverse3d/judges/rubrics/static_object.yaml
@@ -176,6 +176,13 @@ caps:
severity: error
cap: 0.75
note: measured bbox/part contract broken beyond tolerance
+ # a deterministic check the project registered (Spec.checks, tracks/project_checks.py) measured an error
+ - id: project_check
+ when: gate
+ gate: "project:*"
+ severity: error
+ cap: 0.6
+ note: the project's own measurement of the object failed
- id: missing_must_acceptance
when: acceptance
cap: 0.6
diff --git a/harness/codeverse3d/tracks/project_checks.py b/harness/codeverse3d/tracks/project_checks.py
new file mode 100644
index 00000000..ff422a46
--- /dev/null
+++ b/harness/codeverse3d/tracks/project_checks.py
@@ -0,0 +1,77 @@
+"""Project checks (``Spec.checks``): deterministic checks a project registers by import path, run on every round.
+
+The harness gates what it can know about any object — the build, the plan contract, connectivity, budgets. A
+project often knows more: the true geometry of the thing (a surveyed building, a catalogue part), definitions
+of its own (a building's eaves line) and thresholds. A registered check's reports join the round's gates under
+``project:<name>``: an ERROR becomes a refine task with its ``fix_hint``, the judge reads it in the gates section,
+and a rubric ``CapRule(when="gate", gate="project:*")`` caps the score.
+
+A check is ``fn(*, glb, workspace, spec, measurement, round_index, args) -> GateReport | list[GateReport]``. It
+must be deterministic and must not call a model (Law 3). A check that raises costs the round nothing but a WARN
+saying so: a project's bug must not fail or cap the run.
+"""
+
+from __future__ import annotations
+
+import importlib
+import logging
+import time
+from collections.abc import Sequence
+from pathlib import Path
+from typing import Any, Protocol
+
+from codeverse3d.contracts.artifacts import GateFinding, GateReport, Measurement, Severity
+from codeverse3d.contracts.spec import ProjectCheck, Spec
+
+log = logging.getLogger(__name__)
+
+PREFIX = "project:"
+
+
+class CheckFn(Protocol):
+ def __call__(self, *, glb: Path, workspace: Path, spec: Spec, measurement: Measurement | None,
+ round_index: int, args: dict[str, Any]) -> GateReport | Sequence[GateReport]: ...
+
+
+def load_check(entry: str) -> CheckFn:
+ """The function ``entry`` (``package.module:function``) names."""
+ module, _, attr = entry.partition(":")
+ fn = getattr(importlib.import_module(module), attr)
+ if not callable(fn):
+ raise TypeError(f"{entry} is not callable")
+ return fn
+
+
+def _named(report: GateReport, check: ProjectCheck) -> GateReport:
+ """``report`` under ``project:<name>`` (a check returning several reports keeps them apart as
+ ``project:<name>.<its gate>``), its findings renamed with it, so caps and refine keys see one namespace."""
+ own = report.gate or ""
+ if own.startswith(PREFIX):
+ gate = own
+ elif own in ("", check.name):
+ gate = PREFIX + check.name
+ else:
+ gate = f"{PREFIX}{check.name}.{own}"
+ findings = [f.model_copy(update={"gate": gate}) for f in report.findings]
+ return report.model_copy(update={"gate": gate, "findings": findings})
+
+
+def run_project_checks(spec: Spec, workspace: Path, glb: Path, measurement: Measurement | None,
+ round_index: int) -> list[GateReport]:
+ """Every check of ``spec`` on this round's GLB, in the spec's order."""
+ out: list[GateReport] = []
+ for check in spec.checks:
+ t0 = time.time()
+ try:
+ got = load_check(check.entry)(glb=glb, workspace=workspace, spec=spec, measurement=measurement,
+ round_index=round_index, args=dict(check.args))
+ reports = [got] if isinstance(got, GateReport) else list(got)
+ out += [_named(r, check) for r in reports]
+ except Exception as e: # a project's bug must not fail or cap the run
+ log.warning("project check %s failed: %s", check.entry, e)
+ gate = PREFIX + check.name
+ out.append(GateReport.of(gate, [GateFinding(
+ gate=gate, severity=Severity.WARN,
+ message=f"project check {check.entry} raised {type(e).__name__}: {str(e)[:300]}")],
+ duration_ms=int((time.time() - t0) * 1000)))
+ return out
diff --git a/harness/codeverse3d/tracks/steps.py b/harness/codeverse3d/tracks/steps.py
index 992233f1..426f3faa 100644
--- a/harness/codeverse3d/tracks/steps.py
+++ b/harness/codeverse3d/tracks/steps.py
@@ -28,6 +28,7 @@ from codeverse3d.spatial.render import RenderError
from codeverse3d.tracks import skills_hook
from codeverse3d.tracks.common import RunContext, generate_for
from codeverse3d.tracks.generation import GenerationResult, GenerationTask
+from codeverse3d.tracks.project_checks import run_project_checks
from codeverse3d.tracks.repair import RepairOutcome, build_with_repair
from codeverse3d.workspace import Workspace
@@ -189,6 +190,9 @@ def _run_round(
with timed("gates", steps, round_index=index):
rec.measurement = pipeline.measure(ctx, outcome.build)
gates.extend(pipeline.gates(ctx, index, outcome.build, rec.measurement))
+ if ctx.spec.checks and outcome.build.glb_path: # the project's own checks (Spec.checks)
+ gates.extend(run_project_checks(ctx.spec, ctx.ws.root, Path(outcome.build.glb_path),
+ rec.measurement, index))
try:
with timed("render", steps, round_index=index):
rec.renders = (render or pipeline.render)(ctx, index, outcome.build, rec.measurement)
diff --git a/harness/docs/ARCHITECTURE.md b/harness/docs/ARCHITECTURE.md
index d1b0e236..e1ed5e7c 100644
--- a/harness/docs/ARCHITECTURE.md
+++ b/harness/docs/ARCHITECTURE.md
@@ -157,7 +157,8 @@ codeverse3d/
single-shot asset generation), zone_layout.py (per-zone structured layout calls +
deterministic validator), graphics.py (the whole graphics track), plan_features.py
(eval-only: which C3D_* names are live, and pin_plan_blockers() deciding when two arms may
- share one plan), depth.py, skills_hook.py (the round's view of codeverse3d/skills: attach
+ share one plan), depth.py, project_checks.py (Spec.checks: the project's deterministic checks, run on
+ every built round and joined to its gates as project:<name>), skills_hook.py (the round's view of codeverse3d/skills: attach
before generating, probe reads after — a no-op under C3D_SKILLS=0)
record/ what every run WRITES: record.py (finalize_record, load_record, iter_runs,
effective_judgment), deliverable.py (every round's kept build under artifacts/rNN/, and
diff --git a/harness/tests/orchestrator_tracks/test_project_checks.py b/harness/tests/orchestrator_tracks/test_project_checks.py
new file mode 100644
index 00000000..d76ddc06
--- /dev/null
+++ b/harness/tests/orchestrator_tracks/test_project_checks.py
@@ -0,0 +1,78 @@
+"""Spec.checks: a project's deterministic checks join every round's gates as project:<name>."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from codeverse3d.contracts.artifacts import GateFinding, GateReport, Severity
+from codeverse3d.contracts.common import Language, Track
+from codeverse3d.contracts.spec import ProjectCheck, Spec
+from codeverse3d.tracks.project_checks import run_project_checks
+
+HERE = "tests.orchestrator_tracks.test_project_checks"
+
+
+def roof_check(*, glb: Path, workspace: Path, spec: Spec, measurement: Any, round_index: int,
+ args: dict[str, Any]) -> GateReport:
+ within = args["within"]
+ sev = Severity.ERROR if within < 0.85 else Severity.INFO
+ return GateReport.of("", [GateFinding(gate="", severity=sev, target="Roof",
+ message=f"{within:.0%} of the roof within 0.5 m of the survey (round {round_index})",
+ fix_hint="build the roof planes from the survey", data={"kind": "roof_shape"})])
+
+
+def two_reports(**_: Any) -> list[GateReport]:
+ return [GateReport.of("eaves", []), GateReport.of("project:other", [])]
+
+
+def broken(**_: Any) -> GateReport:
+ raise RuntimeError("reference mesh missing")
+
+
+def _spec(*checks: ProjectCheck) -> Spec:
+ return Spec(id="x", track=Track.STATIC_OBJECT, language=Language.THREEJS, prompt="a house", checks=list(checks))
+
+
+def test_checks_are_named_and_imported_by_path() -> None:
+ with pytest.raises(ValueError):
+ ProjectCheck(name="Survey Check", entry=f"{HERE}:roof_check")
+ with pytest.raises(ValueError):
+ ProjectCheck(name="survey", entry="no_function_here")
+ assert "checks" not in _spec().model_dump(mode="json") # a spec without checks serialises as before
+
+
+def test_a_check_joins_the_round_as_project_gate(tmp_path: Path) -> None:
+ spec = _spec(ProjectCheck(name="survey", entry=f"{HERE}:roof_check", args={"within": 0.59}))
+ [rep] = run_project_checks(spec, tmp_path, tmp_path / "object.glb", None, 2)
+ assert rep.gate == "project:survey" and not rep.passed
+ err = rep.errors[0]
+ assert err.gate == "project:survey" and err.data["kind"] == "roof_shape" and "(round 2)" in err.message
+
+ spec = _spec(ProjectCheck(name="survey", entry=f"{HERE}:two_reports"))
+ assert [r.gate for r in run_project_checks(spec, tmp_path, tmp_path / "o.glb", None, 0)] == [
+ "project:survey.eaves", "project:other"]
+
+
+def test_a_broken_check_warns_and_never_fails_the_round(tmp_path: Path) -> None:
+ spec = _spec(ProjectCheck(name="survey", entry=f"{HERE}:broken"),
+ ProjectCheck(name="missing", entry="no_such_module_here:check"))
+ reps = run_project_checks(spec, tmp_path, tmp_path / "o.glb", None, 0)
+ assert [r.gate for r in reps] == ["project:survey", "project:missing"]
+ assert all(r.passed for r in reps)
+ assert "RuntimeError: reference mesh missing" in reps[0].findings[0].message
+ assert reps[1].findings[0].severity is Severity.WARN
+
+
[email protected]("rubric", ["reference", "static_object"])
+def test_a_project_error_caps_the_score(tmp_path: Path, rubric: str) -> None:
+ from codeverse3d.judges.rubrics import apply_caps, load_rubric
+
+ spec = _spec(ProjectCheck(name="survey", entry=f"{HERE}:roof_check", args={"within": 0.59}))
+ gates = run_project_checks(spec, tmp_path, tmp_path / "o.glb", None, 0)
+ assert apply_caps(load_rubric(rubric), 0.9, gates, {}).overall == pytest.approx(0.6)
+ ok = _spec(ProjectCheck(name="survey", entry=f"{HERE}:roof_check", args={"within": 0.97}))
+ assert apply_caps(load_rubric(rubric), 0.9, run_project_checks(ok, tmp_path, tmp_path / "o.glb", None, 0),
+ {}).overall == pytest.approx(0.9)