Skip to content

Official Codebase of 3DCodeVerse

Python

Merge pull request #35 from wqin0/pr/project-checks-gate

gaoypengce2cc97c766a
diff --cc eval/bench/run_bench.py
index 796b919d,88e1849b..f52e2d9e
--- a/eval/bench/run_bench.py
+++ b/eval/bench/run_bench.py
@@@ -55,7 -55,7 +55,8 @@@ from codeverse3d.contracts.common impor
from codeverse3d.contracts.run import RunRecord # noqa: E402
from codeverse3d.contracts.spec import ( # noqa: E402
Constraints,
+ FixedPart,
+ ProjectCheck,
ReferenceImage,
RunOptions,
Spec,
@@@ -86,10 -86,8 +87,12 @@@ class BenchPrompt(BaseModel)
default_factory=list,
description="reference image paths (relative to the battery file); bench/refs/<id>/*.png|jpg are added automatically")
# left out of a dumped battery while empty, so every existing battery serialises as before
+ fixed_parts: list[FixedPart] = Field(
+ default_factory=list, exclude_if=lambda v: not v,
+ description="project-seeded src/ files the run must keep (Spec.fixed_parts); the project seeds them into "
+ "runs/<id>/src/ before the battery runs")
+ checks: list[ProjectCheck] = Field(default_factory=list, exclude_if=lambda v: not v,
+ description="project checks run every round (Spec.checks)")
class Battery(BaseModel):
@@@ -202,7 -200,7 +205,8 @@@ def build_spec
budget=Budget(max_rounds=rounds, max_minutes=max_minutes),
backends=backends, tags=[tag0, battery.name, item.tier, item.category, *extra_tags, *item.tags],
options=RunOptions(variant=variant),
+ fixed_parts=list(item.fixed_parts),
+ checks=list(item.checks),
)
diff --cc eval/tests/test_bench_shared.py
index 7ab60fb0,8a537d57..9b3e9166
--- a/eval/tests/test_bench_shared.py
+++ b/eval/tests/test_bench_shared.py
@@@ -71,22 -71,22 +71,43 @@@ def test_a_run_that_scored_nothing_says
assert row.errors == "" and (row.score_picked, row.picked_round) == (0.6, 0) # one verdict is enough
+def test_a_battery_prompt_carries_fixed_parts_into_the_spec(tmp_path):
+ """A project seeds src/ files per prompt: build_spec must hand them to the run (it builds the Spec from the
+ battery even when the workspace already holds a spec.json), and a prompt without them must serialise exactly
+ as before."""
+ import yaml
+
+ from bench.run_bench import Battery, build_spec
+ from codeverse3d.contracts.common import Backends
+
+ path = tmp_path / "b.yaml"
+ path.write_text(yaml.safe_dump({"name": "b", "track": "static_object", "language": "threejs", "prompts": [
+ {"id": "house", "prompt": "a house", "tier": "hard",
+ "fixed_parts": [{"path": "src/parts/shell.js", "name": "Shell", "bbox_min": [-1, 0, -1], "bbox_max": [1, 2, 1]}]},
+ {"id": "plain", "prompt": "a chair", "tier": "easy"}]}))
+ b = Battery.load(path)
+ spec = build_spec(b, b.prompts[0], backends=Backends(), rounds=1, max_minutes=1, tag0="t")
+ assert [f.path for f in spec.fixed_parts] == ["src/parts/shell.js"]
+ plain = build_spec(b, b.prompts[1], backends=Backends(), rounds=1, max_minutes=1, tag0="t").model_dump(mode="json")
+ assert "fixed_parts" not in plain
++
++
+ def test_a_battery_prompt_carries_checks_into_the_spec(tmp_path):
+ """A project registers checks per prompt: build_spec must hand them to the run (it builds the Spec from the
+ battery even when the workspace already holds a spec.json), and a prompt without them must serialise exactly
+ as before."""
+ import yaml
+
+ from bench.run_bench import Battery, build_spec
+ from codeverse3d.contracts.common import Backends
+
+ path = tmp_path / "b.yaml"
+ path.write_text(yaml.safe_dump({"name": "b", "track": "static_object", "language": "threejs", "prompts": [
+ {"id": "house", "prompt": "a house", "tier": "hard",
+ "checks": [{"name": "survey", "entry": "pkg.mod:check", "args": {"id": "house"}}]},
+ {"id": "plain", "prompt": "a chair", "tier": "easy"}]}))
+ b = Battery.load(path)
+ spec = build_spec(b, b.prompts[0], backends=Backends(), rounds=1, max_minutes=1, tag0="t")
+ assert spec.checks[0].args == {"id": "house"}
+ plain = build_spec(b, b.prompts[1], backends=Backends(), rounds=1, max_minutes=1, tag0="t").model_dump(mode="json")
+ assert "checks" not in plain
diff --cc harness/codeverse3d/contracts/spec.py
index e04648a5,bb1168fa..9e373f45
--- a/harness/codeverse3d/contracts/spec.py
+++ b/harness/codeverse3d/contracts/spec.py
@@@ -27,21 -28,18 +28,33 @@@ class Constraints(BaseModel)
must_not: list[str] = Field(default_factory=list)
+class FixedPart(BaseModel):
+ """A file the PROJECT seeds into ``src/`` that no session may rewrite: measured geometry (a survey shell), the
+ data it reads, a shared library. The planner is told it is given and does not plan it; the agent imports it
+ unchanged (an edit is reverted, like a language's ``HARNESS_OWNED_SRC``); the skeleton keeps it; and when it
+ builds a part, the ``contract_fixed`` gate measures that part against ``bbox_min``/``bbox_max``."""
+
+ path: str = Field(description="workspace-relative file under src/, e.g. 'src/parts/survey_shell.js'")
+ name: str = Field(default="", description="the GLB part it builds (a child of the root, e.g. 'SurveyShell'); "
+ "'' for a data or library file")
+ export: str = Field(default="", description="what the agent imports from it, e.g. 'buildSurveyShell'")
+ bbox_min: Vec3 | None = Field(default=None, description="the built part's box, GLB frame (Y-up, +Z front), m")
+ bbox_max: Vec3 | None = None
+ note: str = Field(default="", description="one line for the planner and the agent")
+
+
+ class ProjectCheck(BaseModel):
+ """A deterministic check the PROJECT registers (``Spec.checks``): ``entry`` names a function
+ (``package.module:function``, importable where the harness runs) that measures each round's GLB against
+ what the project knows — survey geometry, catalogue dimensions — and returns ``GateReport``s. They take part
+ exactly like the built-in gates under the name ``project:<name>``: ERRORs become refine tasks, the judge reads
+ them, and a rubric ``CapRule(when="gate", gate="project:*")`` caps on them (``tracks/project_checks.py``)."""
+
+ name: str = Field(pattern=r"^[a-z0-9][a-z0-9_]*$", description="gate suffix: reports are named project:<name>")
+ entry: str = Field(pattern=r"^[A-Za-z_][\w.]*:[A-Za-z_]\w*$", description="'package.module:function'")
+ args: dict[str, Any] = Field(default_factory=dict, description="keyword data passed to the check as args=")
+
+
class RunOptions(BaseModel):
"""Run-shape options frozen on the spec (distinct from Budget and Backends).
@@@ -63,8 -61,8 +76,10 @@@ class Spec(BaseModel)
references: list[ReferenceImage] = Field(default_factory=list)
constraints: Constraints = Field(default_factory=Constraints)
# left out of spec.json while empty, so every spec without one serialises exactly as before
+ fixed_parts: list[FixedPart] = Field(default_factory=list, exclude_if=lambda v: not v,
+ description="project-seeded files no session may rewrite (FixedPart)")
+ checks: list[ProjectCheck] = Field(default_factory=list, exclude_if=lambda v: not v,
+ description="project checks run on every round's GLB (ProjectCheck)")
budget: Budget = Field(default_factory=Budget)
backends: Backends = Field(default_factory=Backends)
options: RunOptions = Field(default_factory=RunOptions)
diff --cc harness/codeverse3d/judges/rubrics/reference.yaml
index 93bb7d36,a1a25472..592319d3
--- a/harness/codeverse3d/judges/rubrics/reference.yaml
+++ b/harness/codeverse3d/judges/rubrics/reference.yaml
@@@ -119,14 -119,13 +119,21 @@@ caps
kinds: [penetrat, intersect, overlap, collision]
cap: 0.7
measures: [interpenetration]
+ # the project's fixed parts built as declared (Spec.fixed_parts, the contract_fixed gate): a part moved,
+ # scaled or edited past the declared box is the object built wrong, not a detail
+ - id: fixed_part_violation
+ when: gate
+ gate: "contract_fixed"
+ severity: error
+ cap: 0.6
+ note: a project-fixed part is missing or not as declared
+ # a deterministic check the project registered (Spec.checks, tracks/project_checks.py) measured an error
+ - id: project_check
+ when: gate
+ gate: "project:*"
+ severity: error
+ cap: 0.6
+ note: the project's own measurement of the object failed
- id: missing_must_acceptance
when: acceptance
cap: 0.6