Official Codebase of 3DCodeVerse
Python
Merge pull request #35 from wqin0/pr/project-checks-gate
gaoypengce2cc97c766a
diff --cc eval/bench/run_bench.pyindex 796b919d,88e1849b..f52e2d9e--- a/eval/bench/run_bench.py+++ b/eval/bench/run_bench.py@@@ -55,7 -55,7 +55,8 @@@ from codeverse3d.contracts.common imporfrom codeverse3d.contracts.run import RunRecord # noqa: E402from codeverse3d.contracts.spec import ( # noqa: E402Constraints,+ FixedPart,+ ProjectCheck,ReferenceImage,RunOptions,Spec,@@@ -86,10 -86,8 +87,12 @@@ class BenchPrompt(BaseModel)default_factory=list,description="reference image paths (relative to the battery file); bench/refs/<id>/*.png|jpg are added automatically")# left out of a dumped battery while empty, so every existing battery serialises as before+ fixed_parts: list[FixedPart] = Field(+ default_factory=list, exclude_if=lambda v: not v,+ description="project-seeded src/ files the run must keep (Spec.fixed_parts); the project seeds them into "+ "runs/<id>/src/ before the battery runs")+ checks: list[ProjectCheck] = Field(default_factory=list, exclude_if=lambda v: not v,+ description="project checks run every round (Spec.checks)")class Battery(BaseModel):@@@ -202,7 -200,7 +205,8 @@@ def build_specbudget=Budget(max_rounds=rounds, max_minutes=max_minutes),backends=backends, tags=[tag0, battery.name, item.tier, item.category, *extra_tags, *item.tags],options=RunOptions(variant=variant),+ fixed_parts=list(item.fixed_parts),+ checks=list(item.checks),)diff --cc eval/tests/test_bench_shared.pyindex 7ab60fb0,8a537d57..9b3e9166--- a/eval/tests/test_bench_shared.py+++ b/eval/tests/test_bench_shared.py@@@ -71,22 -71,22 +71,43 @@@ def test_a_run_that_scored_nothing_saysassert row.errors == "" and (row.score_picked, row.picked_round) == (0.6, 0) # one verdict is enough+def test_a_battery_prompt_carries_fixed_parts_into_the_spec(tmp_path):+ """A project seeds src/ files per prompt: build_spec must hand them to the run (it builds the Spec from the+ battery even when the workspace already holds a spec.json), and a prompt without them must serialise exactly+ as before."""+ import yaml++ from bench.run_bench import Battery, build_spec+ from codeverse3d.contracts.common import Backends++ path = tmp_path / "b.yaml"+ path.write_text(yaml.safe_dump({"name": "b", "track": "static_object", "language": "threejs", "prompts": [+ {"id": "house", "prompt": "a house", "tier": "hard",+ "fixed_parts": [{"path": "src/parts/shell.js", "name": "Shell", "bbox_min": [-1, 0, -1], "bbox_max": [1, 2, 1]}]},+ {"id": "plain", "prompt": "a chair", "tier": "easy"}]}))+ b = Battery.load(path)+ spec = build_spec(b, b.prompts[0], backends=Backends(), rounds=1, max_minutes=1, tag0="t")+ assert [f.path for f in spec.fixed_parts] == ["src/parts/shell.js"]+ plain = build_spec(b, b.prompts[1], backends=Backends(), rounds=1, max_minutes=1, tag0="t").model_dump(mode="json")+ assert "fixed_parts" not in plain+++++ def test_a_battery_prompt_carries_checks_into_the_spec(tmp_path):+ """A project registers checks per prompt: build_spec must hand them to the run (it builds the Spec from the+ battery even when the workspace already holds a spec.json), and a prompt without them must serialise exactly+ as before."""+ import yaml++ from bench.run_bench import Battery, build_spec+ from codeverse3d.contracts.common import Backends++ path = tmp_path / "b.yaml"+ path.write_text(yaml.safe_dump({"name": "b", "track": "static_object", "language": "threejs", "prompts": [+ {"id": "house", "prompt": "a house", "tier": "hard",+ "checks": [{"name": "survey", "entry": "pkg.mod:check", "args": {"id": "house"}}]},+ {"id": "plain", "prompt": "a chair", "tier": "easy"}]}))+ b = Battery.load(path)+ spec = build_spec(b, b.prompts[0], backends=Backends(), rounds=1, max_minutes=1, tag0="t")+ assert spec.checks[0].args == {"id": "house"}+ plain = build_spec(b, b.prompts[1], backends=Backends(), rounds=1, max_minutes=1, tag0="t").model_dump(mode="json")+ assert "checks" not in plaindiff --cc harness/codeverse3d/contracts/spec.pyindex e04648a5,bb1168fa..9e373f45--- a/harness/codeverse3d/contracts/spec.py+++ b/harness/codeverse3d/contracts/spec.py@@@ -27,21 -28,18 +28,33 @@@ class Constraints(BaseModel)must_not: list[str] = Field(default_factory=list)+class FixedPart(BaseModel):+ """A file the PROJECT seeds into ``src/`` that no session may rewrite: measured geometry (a survey shell), the+ data it reads, a shared library. The planner is told it is given and does not plan it; the agent imports it+ unchanged (an edit is reverted, like a language's ``HARNESS_OWNED_SRC``); the skeleton keeps it; and when it+ builds a part, the ``contract_fixed`` gate measures that part against ``bbox_min``/``bbox_max``."""++ path: str = Field(description="workspace-relative file under src/, e.g. 'src/parts/survey_shell.js'")+ name: str = Field(default="", description="the GLB part it builds (a child of the root, e.g. 'SurveyShell'); "+ "'' for a data or library file")+ export: str = Field(default="", description="what the agent imports from it, e.g. 'buildSurveyShell'")+ bbox_min: Vec3 | None = Field(default=None, description="the built part's box, GLB frame (Y-up, +Z front), m")+ bbox_max: Vec3 | None = None+ note: str = Field(default="", description="one line for the planner and the agent")+++ class ProjectCheck(BaseModel):+ """A deterministic check the PROJECT registers (``Spec.checks``): ``entry`` names a function+ (``package.module:function``, importable where the harness runs) that measures each round's GLB against+ what the project knows — survey geometry, catalogue dimensions — and returns ``GateReport``s. They take part+ exactly like the built-in gates under the name ``project:<name>``: ERRORs become refine tasks, the judge reads+ them, and a rubric ``CapRule(when="gate", gate="project:*")`` caps on them (``tracks/project_checks.py``)."""++ name: str = Field(pattern=r"^[a-z0-9][a-z0-9_]*$", description="gate suffix: reports are named project:<name>")+ entry: str = Field(pattern=r"^[A-Za-z_][\w.]*:[A-Za-z_]\w*$", description="'package.module:function'")+ args: dict[str, Any] = Field(default_factory=dict, description="keyword data passed to the check as args=")++class RunOptions(BaseModel):"""Run-shape options frozen on the spec (distinct from Budget and Backends).@@@ -63,8 -61,8 +76,10 @@@ class Spec(BaseModel)references: list[ReferenceImage] = Field(default_factory=list)constraints: Constraints = Field(default_factory=Constraints)# left out of spec.json while empty, so every spec without one serialises exactly as before+ fixed_parts: list[FixedPart] = Field(default_factory=list, exclude_if=lambda v: not v,+ description="project-seeded files no session may rewrite (FixedPart)")+ checks: list[ProjectCheck] = Field(default_factory=list, exclude_if=lambda v: not v,+ description="project checks run on every round's GLB (ProjectCheck)")budget: Budget = Field(default_factory=Budget)backends: Backends = Field(default_factory=Backends)options: RunOptions = Field(default_factory=RunOptions)diff --cc harness/codeverse3d/judges/rubrics/reference.yamlindex 93bb7d36,a1a25472..592319d3--- a/harness/codeverse3d/judges/rubrics/reference.yaml+++ b/harness/codeverse3d/judges/rubrics/reference.yaml@@@ -119,14 -119,13 +119,21 @@@ capskinds: [penetrat, intersect, overlap, collision]cap: 0.7measures: [interpenetration]+ # the project's fixed parts built as declared (Spec.fixed_parts, the contract_fixed gate): a part moved,+ # scaled or edited past the declared box is the object built wrong, not a detail+ - id: fixed_part_violation+ when: gate+ gate: "contract_fixed"+ severity: error+ cap: 0.6+ note: a project-fixed part is missing or not as declared+ # a deterministic check the project registered (Spec.checks, tracks/project_checks.py) measured an error+ - id: project_check+ when: gate+ gate: "project:*"+ severity: error+ cap: 0.6+ note: the project's own measurement of the object failed- id: missing_must_acceptancewhen: acceptancecap: 0.6