Skip to content

Official Codebase of 3DCodeVerse

Python

Judges: wire-shape-only parse, nine write-only breakdown fields, rubric description/note fields, one judge factory on Services; gallery drops the always-empty stage rows

gaoypengd3851a7c2d4e
diff --git a/harness/codeverse3d/addons/gallery/detail.py b/harness/codeverse3d/addons/gallery/detail.py
index 821dc3be..f8f4b8ec 100644
--- a/harness/codeverse3d/addons/gallery/detail.py
+++ b/harness/codeverse3d/addons/gallery/detail.py
@@ -247,9 +247,8 @@ def _cost_panel(entry: RunEntry, ws: Workspace, rec: RunRecord) -> str:
tele = load_telemetry(ws, rec)
cost = tele.cost if tele is not None else None
if cost is None:
- rows = "".join(_kv(k, f"${v:.4f}") for k, v in sorted(entry.cost_by_stage.items(), key=lambda kv: -kv[1]))
u = rec.total_usage
- rows += "".join([_kv("total", f"${u.cost_usd:.4f}"),
+ rows = "".join([_kv("total", f"${u.cost_usd:.4f}"),
_kv("input tokens", f"{u.input_tokens:,}"), _kv("output tokens", f"{u.output_tokens:,}")])
per_round = "".join(_kv(f"r{r.index} {r.kind}", f"${r.usage.cost_usd:.4f} · {r.minutes:.1f} min")
for r in rec.rounds)
diff --git a/harness/codeverse3d/addons/gallery/index.py b/harness/codeverse3d/addons/gallery/index.py
index 5bcd9cf1..5cba4ecc 100644
--- a/harness/codeverse3d/addons/gallery/index.py
+++ b/harness/codeverse3d/addons/gallery/index.py
@@ -7,7 +7,7 @@ from pathlib import Path
from codeverse3d.addons import select
from codeverse3d.addons.dataset.quality import quality_tier
-from codeverse3d.addons.dataset.sample import gate_error_summary, telemetry_digest
+from codeverse3d.addons.dataset.sample import gate_error_summary
from codeverse3d.addons.gallery.model import (
GalleryIndex,
RootSection,
@@ -172,7 +172,6 @@ def entry_from_record(battery: str, ws: Workspace, rec: RunRecord, *, slug: str
gates = gate_error_summary(rnd)
n_err = sum(gates.values())
minutes = rec.minutes
- digest = telemetry_digest(ws, rec)
caps = rec.extra.get("captions") or {}
hero, hero_label, n_views = hero_view(ws, rec, picked)
plan = rec.plan
@@ -191,8 +190,6 @@ def entry_from_record(battery: str, ws: Workspace, rec: RunRecord, *, slug: str
complexity=cx_index, complexity_band=cx_band, complexity_axes=cx_axes,
sheet=picked_sheet(ws, rec, picked), hero=hero, hero_label=hero_label, n_views=n_views,
links=entry_links(ws, rec, picked), round_rows=round_rows(ws, rec),
- cost_by_stage={k: round(v, 4) for k, v in (digest.get("by_stage") or {}).items()},
- models=dict(digest.get("models") or {}),
)
diff --git a/harness/codeverse3d/addons/gallery/model.py b/harness/codeverse3d/addons/gallery/model.py
index 2af78dce..92ca7cb7 100644
--- a/harness/codeverse3d/addons/gallery/model.py
+++ b/harness/codeverse3d/addons/gallery/model.py
@@ -124,8 +124,6 @@ class RunEntry(BaseModel):
n_views: int = Field(default=0, description="how many individual views the picked round has")
links: list[RunLink] = Field(default_factory=list)
round_rows: list[RoundRow] = Field(default_factory=list)
- cost_by_stage: dict[str, float] = Field(default_factory=dict)
- models: dict[str, str] = Field(default_factory=dict)
@property
def key(self) -> str:
diff --git a/harness/codeverse3d/judges/rubrics.py b/harness/codeverse3d/judges/rubrics.py
index da61e9ad..5a18ac9b 100644
--- a/harness/codeverse3d/judges/rubrics.py
+++ b/harness/codeverse3d/judges/rubrics.py
@@ -86,8 +86,6 @@ class DefectItem(BaseModel):
text: str = Field(description="concrete, visually checkable statement of the defect")
penalty: float = Field(default=0.0, ge=0.0, le=1.0)
cap: float | None = Field(default=None, ge=0.0, le=1.0)
- note: str = ""
-
@field_validator("id")
@classmethod
def _ident(cls, v: str) -> str:
@@ -124,8 +122,6 @@ class CapRule(BaseModel):
"{depth_m: VETO_PENETRATION_DEPTH_M}. scene states {gap_m: 0.02} / {sunk_m: 0.10} / "
"{overlap: 0.20}; 0.0 = any WARN of the kind that carries the key",
)
- note: str = ""
-
@model_validator(mode="after")
def _default_measures(self) -> CapRule:
if not self.measures:
@@ -141,7 +137,6 @@ class Rubric(BaseModel):
caps: list[CapRule] = Field(default_factory=list)
defects: list[DefectItem] = Field(default_factory=list, description="binary checklist scored in code")
extra_instructions: str = ""
- description: str = ""
@model_validator(mode="after")
def _consistent(self) -> Rubric:
@@ -308,57 +303,22 @@ def wire_schema(rubric: Rubric, acceptance_ids: list[str]) -> dict[str, Any]:
return build_wire_model(rubric, acceptance_ids).model_json_schema(by_alias=True)
-def _records_to_dict(value: Any, key_names: tuple[str, ...]) -> dict[str, Any]:
- """Accept ``{id: {...}}`` or ``[{id: ..., ...}, ...]`` and return ``{id: {...}}``."""
- if isinstance(value, dict):
- return value
- if isinstance(value, list):
- out: dict[str, Any] = {}
- for rec in value:
- if not isinstance(rec, dict):
- raise JudgeParseError(f"record is not an object: {rec!r}")
- key = next((rec[k] for k in key_names if k in rec), None)
- if key is None:
- raise JudgeParseError(f"record lacks an id field {key_names}: {rec!r}")
- out[str(key)] = {k: v for k, v in rec.items() if k not in key_names}
- return out
- raise JudgeParseError(f"expected object or list, got {type(value).__name__}")
-
-
-def _coerce_acceptance(value: Any) -> dict[str, Any]:
- recs = _records_to_dict(value, ("id", "item_id", "item"))
- out: dict[str, Any] = {}
- for k, v in recs.items():
- if isinstance(v, bool):
- out[k] = {"verified": v, "evidence": ""}
- elif isinstance(v, dict):
- vv = dict(v)
- if "verified" not in vv and "passed" in vv:
- vv["verified"] = vv.pop("passed")
- out[k] = vv
- else:
- raise JudgeParseError(f"acceptance item {k!r}: unexpected value {v!r}")
- return out
+def _object(value: Any, what: str) -> dict[str, Any]:
+ if not isinstance(value, dict):
+ raise JudgeParseError(f"{what} is not an object: {type(value).__name__}")
+ return value
-def _coerce_defects(value: Any) -> tuple[dict[str, bool], dict[str, str]]:
- """``{id: bool}`` or ``{id: {present, evidence}}`` or a list of records → (present, evidence)."""
- if value is None:
- return {}, {}
- recs = _records_to_dict(value, ("id", "defect", "item"))
+def _flatten_defects(value: Any) -> tuple[dict[str, bool], dict[str, str]]:
+ """``{id: {present, evidence}}`` (the wire shape) → (present, evidence); ``present`` must be a real boolean."""
present: dict[str, bool] = {}
evidence: dict[str, str] = {}
- for k, v in recs.items():
- if isinstance(v, bool):
- present[k] = v
- elif isinstance(v, dict):
- flag = v.get("present", v.get("value", v.get("verified")))
- if not isinstance(flag, bool):
- raise JudgeParseError(f"defect {k!r}: 'present' must be a boolean, got {flag!r}")
- present[k] = flag
- evidence[k] = str(v.get("evidence", "") or "")
- else:
- raise JudgeParseError(f"defect item {k!r}: unexpected value {v!r}")
+ for k, v in _object(value, "defects").items():
+ flag = v.get("present") if isinstance(v, dict) else None
+ if not isinstance(flag, bool):
+ raise JudgeParseError(f"defect {k!r}: 'present' must be a boolean, got {v!r}")
+ present[k] = flag
+ evidence[k] = str(v.get("evidence", "") or "")
return present, evidence
@@ -373,23 +333,16 @@ def parse_judge_output(
raise JudgeParseError(f"no JSON object in reply: {e}") from e
if not isinstance(payload, dict):
raise JudgeParseError(f"reply is not a JSON object: {type(payload).__name__}")
- data = dict(payload)
- if "criteria" not in data and "per_criterion" in data:
- data["criteria"] = data.pop("per_criterion")
+ data = payload
try:
- criteria = _records_to_dict(data.get("criteria", {}), ("id", "criterion", "name"))
- criteria = {
- k: ({"score": v, "evidence": ""} if isinstance(v, (int, float)) else v) for k, v in criteria.items()
- }
- acceptance = _coerce_acceptance(data.get("acceptance", {}))
- defects, defect_evidence = _coerce_defects(data.get("defects", {}))
+ defects, defect_evidence = _flatten_defects(data.get("defects") or {})
out = JudgeOutput(
- criteria=criteria,
+ criteria=_object(data.get("criteria", {}), "criteria"),
summary=str(data.get("summary", "") or ""),
strengths=[str(s) for s in data.get("strengths", []) or []],
issues=data.get("issues", []) or [],
improvement_plan=data.get("improvement_plan", []) or [],
- acceptance=acceptance,
+ acceptance=_object(data.get("acceptance", {}), "acceptance"),
defects=defects,
defect_evidence=defect_evidence,
)
@@ -404,9 +357,8 @@ def parse_judge_output(
extra = [k for k in out.criteria if k not in rubric.weights]
for k in extra:
out.criteria.pop(k)
- missing_acc = [a for a in acceptance_ids if a not in out.acceptance]
- for a in missing_acc:
- out.acceptance[a] = AcceptanceVerdict(verified=False, evidence="not answered by judge")
+ out.acceptance = {a: out.acceptance.get(a) or AcceptanceVerdict(verified=False, evidence="not answered by judge")
+ for a in acceptance_ids}
known = {d.id for d in rubric.defects}
for k in [k for k in out.defects if k not in known]:
out.defects.pop(k)
@@ -611,22 +563,10 @@ A stored verdict is reproducible from its samples only under the version that wr
it (``eval/bench/rejudge_offline.py --identity`` compares nothing else); an older stamp is
drift to report, not a fault.
-0 — before 2026-08-26 (no stamp on disk): majority vote, penalties, the cap ladder.
-1 — 2026-08-26, 7a9b6d3: ``veto_measured_defects`` — a checklist defect a passed gate
- measured absent neither penalises nor caps. (Unstamped; 41 corpus verdicts predate
- it and move on replay.)
-2 — 2026-08-30: ``CapRule.measures`` (the veto reaches ``interpenetration``), the object
- rubrics' floating/penetration rules watch ``connectivity`` alone, a defect vote tie
- is absent, ``missing_must_acceptance`` is graded, ``overridden`` + this stamp in raw.
-3 — 2026-09-24: a gate cap rule's ``kinds`` match the finding's typed ``data.kind`` /
- ``data.code``; the message only when it has neither, the target never (N50). A replay
- of the 1 229 recorded judged rounds moved no cap and no veto.
-4 — 2026-10-14 (scene): the measured-absent veto is scoped to the rule's ``kinds`` — a
- watched report that failed on an unrelated kind no longer cancels it — and the WARN
- magnitude that keeps a claim alive is the rule's ``veto_warn_keys`` (the object rubrics
- keep the ``depth_m`` ≥ 8 mm default, so their verdicts replay unchanged unless the
- connectivity report failed on a penetration while the judge claimed floating, or the
- reverse).
+Versions: 1 veto of checklist defects a passed gate measured absent · 2 ``CapRule.measures``, a defect
+vote tie is absent, graded ``missing_must_acceptance`` · 3 cap ``kinds`` match the typed ``data.kind`` /
+``data.code`` (a replay of 1 229 judged rounds moved no cap and no veto) · 4 the veto is scoped to the
+rule's ``kinds`` and its WARN magnitude is ``veto_warn_keys`` (history: ``eval/docs/EVAL.md``).
"""
@@ -635,32 +575,20 @@ class ScoreBreakdown(BaseModel):
status: str = Field(default="ok", description="ok | degraded")
scoring_version: int = Field(default=0, description="SCORING_VERSION that produced this breakdown")
- per_criterion_mean: dict[str, float]
per_criterion_std: dict[str, float]
- per_sample_overall: list[float]
overall_uncapped: float = Field(
description="rubric-weighted mean of the per-criterion means, before defects/caps"
)
- overall_after_defects: float = Field(
- default=0.0, description="overall_uncapped minus defect penalties"
- )
overall: float
- floors_hit: list[dict[str, Any]] = Field(default_factory=list)
caps: CapResult
- must_missing: list[str] = Field(default_factory=list)
- acceptance_votes: dict[str, list[bool]] = Field(default_factory=dict)
defects: dict[str, bool] = Field(
default_factory=dict, description="checklist id → present (majority vote)"
)
- defect_votes: dict[str, list[bool]] = Field(default_factory=dict)
overridden: list[str] = Field(default_factory=list,
description="checklist defects the vote marked present that a passed gate measured absent (veto_measured_defects)")
defect_penalty: float = 0.0
n_requested: int
- n_used: int
sample_errors: list[str] = Field(default_factory=list)
- tie_broken: list[str] = Field(default_factory=list,
- description="ids an exact vote tie decided: acceptance → the representative sample, defect → absent")
rubric_hash: str = ""
judge_prompt_hash: str = Field(default="", description="hash of everything the judge is told that is constant per rubric: "
"role prompt + view-rig rules + wire schema (prompt_builder.judge_prompt_hash)")
@@ -676,17 +604,15 @@ def _representative(samples: list[JudgeOutput], overalls: list[float]) -> JudgeO
return min(zip(samples, overalls, strict=True), key=lambda so: abs(so[1] - mean))[0]
-def _vote(votes: list[bool], tie_break: bool) -> tuple[bool, bool]:
- """Majority of ``votes``; an exact tie takes ``tie_break``. Returns (decision, was_tie).
+def _vote(votes: list[bool], tie_break: bool) -> bool:
+ """Majority of ``votes``; an exact tie takes ``tie_break``.
Acceptance ties take the representative sample; defect ties take ``False`` (the defect bears the burden).
"""
if not votes:
- return False, False
+ return False
yes = sum(1 for v in votes if v)
- if yes * 2 == len(votes):
- return bool(tie_break), True
- return yes * 2 > len(votes), False
+ return bool(tie_break) if yes * 2 == len(votes) else yes * 2 > len(votes)
def aggregate_samples(
@@ -722,7 +648,6 @@ def aggregate_samples(
for cid, sc, fl in rubric.floors_hit(mean_scores)
]
rep = _representative(samples, overalls)
- tie_broken: list[str] = []
votes: dict[str, list[bool]] = {}
for s in samples:
for aid, ok in s.acceptance_bools.items():
@@ -731,9 +656,7 @@ def aggregate_samples(
votes.setdefault(a.id, [False])
acceptance: dict[str, bool] = {}
for aid, v in votes.items():
- acceptance[aid], tied = _vote(v, bool(rep.acceptance_bools.get(aid, False)))
- if tied:
- tie_broken.append(aid)
+ acceptance[aid] = _vote(v, bool(rep.acceptance_bools.get(aid, False)))
must_missing = missing_must_items(acceptance_items, acceptance)
d_votes: dict[str, list[bool]] = {d.id: [] for d in rubric.defects}
@@ -742,9 +665,7 @@ def aggregate_samples(
d_votes.setdefault(did, []).append(bool(flag))
defects: dict[str, bool] = {}
for did, v in d_votes.items():
- defects[did], tied = _vote(v, False)
- if tied:
- tie_broken.append(did)
+ defects[did] = _vote(v, False)
# a defect a passed gate measured as absent neither penalises nor caps (``veto_measured_defects``)
defects, overridden = veto_measured_defects(rubric, defects, gates)
penalty = round(sum(rubric.defect(did).penalty for did, on in defects.items() if on), 4)
@@ -780,24 +701,15 @@ def aggregate_samples(
breakdown = ScoreBreakdown(
scoring_version=SCORING_VERSION,
- per_criterion_mean=mean_scores,
per_criterion_std=std_scores,
- per_sample_overall=[round(o, 4) for o in overalls],
overall_uncapped=round(overall_uncapped, 4),
- overall_after_defects=round(after_defects, 4),
overall=round(overall, 4),
- floors_hit=floors,
caps=caps,
- must_missing=must_missing,
- acceptance_votes=votes,
defects=defects,
- defect_votes=d_votes,
overridden=overridden,
defect_penalty=penalty,
n_requested=n_requested or len(samples),
- n_used=len(samples),
sample_errors=sample_errors or [],
- tie_broken=tie_broken,
rubric_hash=rubric.content_hash(),
judge_prompt_hash=judge_prompt_hash,
samples=[s.model_dump(mode="json") for s in samples],
@@ -866,7 +778,6 @@ def degraded_judgment(
"scoring_version": SCORING_VERSION,
"error": error,
"n_requested": n_requested,
- "n_used": 0,
"rubric_hash": rubric.content_hash(),
"judge_prompt_hash": judge_prompt_hash,
}
diff --git a/harness/codeverse3d/judges/rubrics/articulated.yaml b/harness/codeverse3d/judges/rubrics/articulated.yaml
index 54478b61..efa348a5 100644
--- a/harness/codeverse3d/judges/rubrics/articulated.yaml
+++ b/harness/codeverse3d/judges/rubrics/articulated.yaml
@@ -1,12 +1,6 @@
name: articulated
version: 1
pass_threshold: 0.7
-description: 'Rubric for an ARTICULATED object (cabinet with doors/drawers, laptop, folding chair, robot arm, scissors...)
- judged on the rest-pose render rig PLUS the POSE SHEET: a harness-made grid whose rows are poses (rest, then each
- joint at a swept value, labelled ''<joint>@<value>'') and whose columns are views (3/4, back 3/4, front), with
- optional 2x2 montages of pose_* tiles. The pose sheet is REQUIRED: without it the overall is capped (see caps)
- because articulation cannot be judged. Static criteria are inherited from static_object at reduced weight;
- articulation criteria score whether the joints are real, placed right and move plausibly.'
extra_instructions: 'POSE tiles (the POSE SHEET image and any tile whose label starts with ''pose_'' or names a
joint@value) show the SAME object with joints moved by the harness (forward kinematics from the authored URDF);
''rest'' is the rest pose, ''<joint>@upper'' / ''@lower'' / ''@mid'' the joint at that limit or midpoint. Compare
@@ -158,13 +152,13 @@ caps:
gate: '*urdf*'
severity: error
cap: 0.3
- note: URDF does not parse / links unresolved / joint tree invalid
+ # URDF does not parse / links unresolved / joint tree invalid
- id: contract_violation
when: gate
gate: contract*
severity: error
cap: 0.75
- note: plan part missing or measured bbox broken beyond tolerance
+ # plan part missing or measured bbox broken beyond tolerance
- id: joint_collision
when: gate
gate: '*'
@@ -175,7 +169,7 @@ caps:
- intersect
- self_collision
cap: 0.6
- note: a swept pose collides link-vs-link
+ # a swept pose collides link-vs-link
- id: floating_part
when: gate
gate: '*'
@@ -189,8 +183,8 @@ caps:
- island
- touches nothing
cap: 0.6
- note: 'stays on gate "*" (2026-08-30): connectivity reports rest-pose floating and joint_sweep
- "touches nothing", so unlike the object rubrics no single gate owns this measurement'
+ # stays on gate "*" (2026-08-30): connectivity reports rest-pose floating and joint_sweep "touches
+ # nothing", so unlike the object rubrics no single gate owns this measurement
- id: penetration_error
when: gate
gate: '*'
@@ -203,15 +197,15 @@ caps:
measures:
- interpenetration
- pose_clips_body
- note: 'stays on gate "*": connectivity can report a rest-pose interpenetration and joint_sweep
- collides every link pair in every sampled pose, the rest pose included
- (spatial/joints_sweep.sweep_collisions). The veto therefore needs EVERY gate clean —
- conservative, and sound for both the rest-pose and the posed checklist claims'
+ # stays on gate "*": connectivity can report a rest-pose interpenetration and joint_sweep collides
+ # every link pair in every sampled pose, the rest pose included
+ # (spatial/joints_sweep.sweep_collisions). The veto therefore needs EVERY gate clean —
+ # conservative, and sound for both the rest-pose and the posed checklist claims
- id: missing_must_acceptance
when: acceptance
cap: 0.6
graded: true
- note: 0.6 + 0.4 * verified/total must items (2026-08-30, as static_object); pass/fail unchanged
+ # 0.6 + 0.4 * verified/total must items (2026-08-30, as static_object); pass/fail unchanged
- id: unattached_link
when: gate
gate: '*'
@@ -219,7 +213,7 @@ caps:
kinds:
- unattached
cap: 0.6
- note: a link is not physically attached to its parent as its joint type requires
+ # a link is not physically attached to its parent as its joint type requires
- id: missing_pose_sheet
when: missing_views
kinds:
@@ -227,7 +221,7 @@ caps:
- pose_sheet
- pose_
cap: 0.5
- note: the articulated rubric requires posed views; without them articulation is unjudged
+ # the articulated rubric requires posed views; without them articulation is unjudged
defects:
- id: wrong_object
text: The renders show a different object category than the brief (or nothing recognisable).
diff --git a/harness/codeverse3d/judges/rubrics/asset.yaml b/harness/codeverse3d/judges/rubrics/asset.yaml
index 40f7a91c..4078aaee 100644
--- a/harness/codeverse3d/judges/rubrics/asset.yaml
+++ b/harness/codeverse3d/judges/rubrics/asset.yaml
@@ -1,12 +1,6 @@
name: asset
version: 1
pass_threshold: 0.70
-description: >
- Per-asset rubric used inside scenes: ONE asset (a building, a tree, a boat, a lamp
- post...) isolated and rendered on the object rig. Stricter on silhouette / all-sides
- completeness than the static-object rubric, lighter on the brief because the asset
- is only described by a one-line plan entry.
-
extra_instructions: >
The asset is judged as a scene ingredient: it must read correctly at a distance
(silhouette) AND be finished on every side because cameras will see it from
@@ -141,7 +135,7 @@ caps:
when: acceptance
cap: 0.6
graded: true
- note: 0.6 + 0.4 * verified/total must items (2026-08-30, as static_object); pass/fail unchanged
+ # 0.6 + 0.4 * verified/total must items (2026-08-30, as static_object); pass/fail unchanged
# Binary checklist (judge answers, code scores: penalty subtracted, cap applied).
defects:
diff --git a/harness/codeverse3d/judges/rubrics/reference.yaml b/harness/codeverse3d/judges/rubrics/reference.yaml
index 592319d3..52c90aff 100644
--- a/harness/codeverse3d/judges/rubrics/reference.yaml
+++ b/harness/codeverse3d/judges/rubrics/reference.yaml
@@ -1,12 +1,6 @@
name: reference
version: 1
pass_threshold: 0.70
-description: >
- Image-conditioned rubric: the object must match one or more REFERENCE images. The
- silhouette criterion is MEASURED in code (IoU of the front render vs the target
- reference) and shown to the judge; the judge scores the perceptual criteria by
- comparing reference images to renders side by side.
-
extra_instructions: >
Images labelled REFERENCE are the target the object must reproduce; images labelled
VIEW are renders of the generated object. Compare like with like (the front render
@@ -126,19 +120,19 @@ caps:
gate: "contract_fixed"
severity: error
cap: 0.6
- note: a project-fixed part is missing or not as declared
+ # a project-fixed part is missing or not as declared
# a deterministic check the project registered (Spec.checks, tracks/project_checks.py) measured an error
- id: project_check
when: gate
gate: "project:*"
severity: error
cap: 0.6
- note: the project's own measurement of the object failed
+ # the project's own measurement of the object failed
- id: missing_must_acceptance
when: acceptance
cap: 0.6
graded: true
- note: 0.6 + 0.4 * verified/total must items (2026-08-30, as static_object); pass/fail unchanged
+ # 0.6 + 0.4 * verified/total must items (2026-08-30, as static_object); pass/fail unchanged
# Binary checklist (judge answers, code scores: penalty subtracted, cap applied).
defects:
diff --git a/harness/codeverse3d/judges/rubrics/scene.yaml b/harness/codeverse3d/judges/rubrics/scene.yaml
index 52d0d0f5..6e068946 100644
--- a/harness/codeverse3d/judges/rubrics/scene.yaml
+++ b/harness/codeverse3d/judges/rubrics/scene.yaml
@@ -1,22 +1,6 @@
name: scene
version: 2
pass_threshold: 0.70
-description: >
- Rubric for a procedural 3D SCENE (a place) judged on its authored cameras plus a
- fitted overview rig, at one or more animation times, with a probe digest (console/shader
- errors, census) and diagnostic CONTACT CROPS at the placement findings the gate measured.
- The scene owns its lighting, sky and fog: grade the picture it makes, and the place it
- depicts. v2 (2026-10-14, the argon scene-tuning program): stricter and detail-aware —
- floating and sunken are separate measured defects, interpenetration / scale / hard-edged
- particles / missing planned elements / hero placement join the checklist, the caps follow
- the scene_placement gate's typed kinds, and every criterion is anchored at six levels so the
- judge uses the whole scale.
- 2026-10-15 (owner's rule — the harness states requirements, never templates): the craft
- criteria and defects judge each place against what ITS brief implies instead of one recipe
- (blended ground + paths, clustered scatter, foreground-frame / midground / horizon, fog and
- a two-hue key/fill everywhere), and `diorama_look` names the template itself — a block on a
- plate ringed by a uniform backdrop with nothing beyond — as a defect.
-
extra_instructions: >
Views named overview_* are HARNESS cameras fitted to the scene bounds — nobody framed
them; use them only for layout, coverage, floating / sunken / clipping assets and scale,
@@ -354,7 +338,7 @@ caps:
measures: [floating_asset]
cap: 0.60
veto_warn_keys: {gap_m: 0.02} # = Placement.floating_indoor_m (= contact_m): a WARN of the kind always reaches it
- note: a gap over 0.08 m, or more than two floaters, is an ERROR (Settings.placement, D105)
+ # a gap over 0.08 m, or more than two floaters, is an ERROR (Settings.placement, D105)
- id: sunken_asset
when: gate
gate: "scene_placement"
@@ -385,7 +369,7 @@ caps:
severity: error
kinds: [free_exempt]
cap: 0.75
- note: more than half the placed assets tagged placement='free' — the author hid them from the gate
+ # more than half the placed assets tagged placement='free' — the author hid them from the gate
- id: build_error
when: gate
gate: "build*"
@@ -400,14 +384,15 @@ caps:
- id: console_error
when: console
cap: 0.6
- note: runtime JS errors seen while rendering
+ # runtime JS errors seen while rendering
- id: camera_in_geometry
when: gate
gate: "*"
severity: error
kinds: [camera_in_geometry, camera_underground, camera_blocked]
cap: 0.5
- note: the kinds the scene_frames gate really emits (frame_metrics.py); the v1 list named five that nothing emitted
+ # the kinds the scene_frames gate really emits (frame_metrics.py); the v1 list named five that
+ # nothing emitted
# scene_frames gate (codeverse3d.spatial.frame_metrics): deterministic frame quality of the
# authored cameras — a crushed dusk, a blown sky, a one-colour frame or an establishing shot
# that shows a tiny island of content can never score as a finished picture.
@@ -417,32 +402,31 @@ caps:
severity: error
kinds: [dark_frame]
cap: 0.55
- note: mean luminance < 0.12 or > 35 % near-black pixels on an authored camera
+ # mean luminance < 0.12 or > 35 % near-black pixels on an authored camera
- id: blown_frame
when: gate
gate: "scene_frames"
severity: error
kinds: [blown_frame]
cap: 0.6
- note: "> 20 % pure-white pixels on an authored camera"
+ # > 20 % pure-white pixels on an authored camera
- id: flat_frame
when: gate
gate: "scene_frames"
severity: error
kinds: [flat_frame]
cap: 0.65
- note: one luminance band holds > 92 % of an authored frame
+ # one luminance band holds > 92 % of an authored frame
- id: content_small
when: gate
gate: "scene_frames"
severity: error
kinds: [content_small]
cap: 0.6
- note: establishing shot covers < 20 % content (rest is sky/ground)
+ # establishing shot covers < 20 % content (rest is sky/ground)
- id: missing_must_acceptance
when: acceptance
cap: 0.6
graded: true
- note: >
- 0.6 + 0.4 * verified/total over the must items, as on the object tracks since
- 2026-08-30; the verdict still FAILS on any unverified must item.
+ # 0.6 + 0.4 * verified/total over the must items, as on the object tracks since 2026-08-30; the
+ # verdict still FAILS on any unverified must item.
diff --git a/harness/codeverse3d/judges/rubrics/shader.yaml b/harness/codeverse3d/judges/rubrics/shader.yaml
index d88918d1..d40094ff 100644
--- a/harness/codeverse3d/judges/rubrics/shader.yaml
+++ b/harness/codeverse3d/judges/rubrics/shader.yaml
@@ -1,16 +1,6 @@
name: shader
version: 2
pass_threshold: 0.70
-description: >
- Rubric for the GRAPHICS track (Shadertoy-style fragment shader or raw OpenGL program), judged on
- a contact sheet of frames sampled at t = 0, 1, 2.5, 4, 6 s plus the harness frame metrics.
- v2 (2026-08-26) replaces "brief fidelity" with LIKENESS: does the effect look like the thing
- the brief names — the physics of it, not the list of nouns. Measured on 17 judged runs before
- this revision: Spearman(judge, a curator's eye) = 0.16, judge mean 0.77 against an eye mean of
- 0.55; three aurora versions that a person would not accept as an aurora scored 0.78-0.94 with
- empty issue lists because every noun in the brief was present. The checklist below names the
- ways those frames failed so that a 0.9 is once again something a person would screenshot.
-
extra_instructions: >
Every view is the SAME program rendered at a different time (label t=...s); there are no camera
angles. Compare the frames to judge motion: identical frames mean the effect is static; wildly
@@ -131,7 +121,7 @@ caps:
gate: "build*"
severity: error
cap: 0.0
- note: the shader/program did not compile or render; nothing to score
+ # the shader/program did not compile or render; nothing to score
- id: lint_error
when: gate
gate: "lint*"
@@ -143,7 +133,7 @@ caps:
severity: error
kinds: [nan]
cap: 0.0
- note: NaN/Inf pixels detected in the float readback
+ # NaN/Inf pixels detected in the float readback
- id: black_or_blown
when: gate
gate: "gl_frames"
@@ -156,7 +146,7 @@ caps:
severity: warn
kinds: [static]
cap: 0.5
- note: motion was planned but frames do not change over time
+ # motion was planned but frames do not change over time
- id: flicker
when: gate
gate: "gl_frames"
@@ -166,9 +156,8 @@ caps:
- id: missing_must_acceptance
when: acceptance
cap: 0.7
- note: >
- planner-written must items are fallible on this track (a stylised wave lost 0.25 to three of
- them while looking better than anything else in the battery); the cap is 0.7, not 0.6
+ # planner-written must items are fallible on this track (a stylised wave lost 0.25 to three of
+ # them while looking better than anything else in the battery); the cap is 0.7, not 0.6
defects:
# black_or_blown / static_frames / flicker share their id with the gl_frames cap rule above on
diff --git a/harness/codeverse3d/judges/rubrics/static_object.yaml b/harness/codeverse3d/judges/rubrics/static_object.yaml
index 59df000f..0b08880a 100644
--- a/harness/codeverse3d/judges/rubrics/static_object.yaml
+++ b/harness/codeverse3d/judges/rubrics/static_object.yaml
@@ -1,11 +1,6 @@
name: static_object
version: 1
pass_threshold: 0.72
-description: >
- Assembly-level rubric for ONE static object (furniture, tool, vehicle, appliance,
- plant, creature, building...) judged on a labelled multi-view render rig plus a
- measurement table. Scores are per criterion; the overall is computed in code.
-
extra_instructions: >
Judge what is VISIBLE in the montages and what the measurement table states; do not
credit detail you cannot see and do not invent faults. Use the underside / low
@@ -152,7 +147,7 @@ caps:
gate: "build*"
severity: error
cap: 0.0
- note: the code did not build or export; nothing to score
+ # the code did not build or export; nothing to score
# Both watch `connectivity` alone (2026-08-30): over 356 static_object records it is the
# only gate that emits floating / penetration ERRORs (540 findings), so the cap loses
# nothing — and the veto (judges/rubrics.measured_absent) stops being switched off by a
@@ -175,22 +170,21 @@ caps:
gate: "contract*"
severity: error
cap: 0.75
- note: measured bbox/part contract broken beyond tolerance
+ # measured bbox/part contract broken beyond tolerance
# a deterministic check the project registered (Spec.checks, tracks/project_checks.py) measured an error
- id: project_check
when: gate
gate: "project:*"
severity: error
cap: 0.6
- note: the project's own measurement of the object failed
+ # the project's own measurement of the object failed
- id: missing_must_acceptance
when: acceptance
cap: 0.6
graded: true
- note: >
- 0.6 + 0.4 * verified/total over the must items (2026-08-30): the flat 0.6 was the
- decisive cap on 121 of 424 verdicts (28.6 %), one unverified item scoring like ten.
- The verdict still FAILS on any unverified must item; only the score keeps its gradient.
+ # 0.6 + 0.4 * verified/total over the must items (2026-08-30): the flat 0.6 was the decisive cap
+ # on 121 of 424 verdicts (28.6 %), one unverified item scoring like ten. The verdict still FAILS
+ # on any unverified must item; only the score keeps its gradient.
# Binary checklist: answered by the judge, scored in code (penalty subtracted, cap applied).
defects:
diff --git a/harness/codeverse3d/tracks/common.py b/harness/codeverse3d/tracks/common.py
index c74eb010..9a9a4161 100644
--- a/harness/codeverse3d/tracks/common.py
+++ b/harness/codeverse3d/tracks/common.py
@@ -47,16 +47,11 @@ class Services:
def coding_agent(self, agent_id: str) -> Any:
return _import("codeverse3d.agents", "get_coding_agent")(agent_id)
- def judge(self, rubric: str, model_id: str, n_samples: int = 1) -> Any:
- return _import("codeverse3d.judges.vlm_judge", "VlmJudge")(rubric=rubric, model_id=model_id, n_samples=n_samples)
-
- def reference_judge(self, model_id: str, n_samples: int = 1, rubric: str = "reference") -> Any:
- """Image-conditioned judge (spec has reference images): renders + references + silhouette IoU."""
- return _import("codeverse3d.judges.vlm_judge", "ReferenceJudge")(model_id=model_id, n_samples=n_samples, rubric=rubric)
-
- def likeness_judge(self, model_id: str, n_samples: int = 1, rubric: str = "shader") -> Any:
- """Reference photos beside the frames, no silhouette (graphics / scene with images)."""
- return _import("codeverse3d.judges.vlm_judge", "LikenessJudge")(model_id=model_id, n_samples=n_samples, rubric=rubric)
+ def judge(self, track: Track, rubric: str, model_id: str, n_samples: int = 1, *, references: bool = False) -> Any:
+ """The judge that scores a round: the class ``judge_for`` picks, shared with every replay."""
+ judges = importlib.import_module("codeverse3d.judges.vlm_judge")
+ cls = judges.judge_for(track, rubric, references=references)
+ return cls(model_id=model_id, n_samples=n_samples, rubric=rubric)
def runtime(self, language: Language) -> Any:
return _import("codeverse3d.languages", "get_runtime")(language)
diff --git a/harness/codeverse3d/tracks/lifecycle.py b/harness/codeverse3d/tracks/lifecycle.py
index 805b9d89..8856e29f 100644
--- a/harness/codeverse3d/tracks/lifecycle.py
+++ b/harness/codeverse3d/tracks/lifecycle.py
@@ -429,15 +429,9 @@ class BaseTrack:
"""The main judge: the injected one, else an instance of the class ``judge_for`` picks for this track."""
if self._judge is not None:
return self._judge
- from codeverse3d.judges.vlm_judge import LikenessJudge, ReferenceJudge, judge_for
-
- n_samples = ctx.policy.judge_samples if n_samples is None else n_samples
- cls = judge_for(ctx.track, ctx.rubric, references=bool(ctx.spec.references))
- if cls is LikenessJudge:
- return self.services.likeness_judge(ctx.spec.backends.judge, n_samples=n_samples, rubric=ctx.rubric)
- if cls is ReferenceJudge:
- return self.services.reference_judge(ctx.spec.backends.judge, n_samples=n_samples, rubric=ctx.rubric)
- return self.services.judge(ctx.rubric, ctx.spec.backends.judge, n_samples=n_samples)
+ return self.services.judge(ctx.track, ctx.rubric, ctx.spec.backends.judge,
+ ctx.policy.judge_samples if n_samples is None else n_samples,
+ references=bool(ctx.spec.references))
def after_plan(self, ctx: RunContext) -> None:
"""Bind judge / generator backends once the plan exists."""
diff --git a/harness/codeverse3d/tracks/scene_assets.py b/harness/codeverse3d/tracks/scene_assets.py
index 95f327a2..b5166754 100644
--- a/harness/codeverse3d/tracks/scene_assets.py
+++ b/harness/codeverse3d/tracks/scene_assets.py
@@ -533,7 +533,7 @@ def _judge_and_fix(ctx: RunContext, asset: AssetPlan, result: AssetResult, rende
events = ctx.events
try:
- judge = ctx.services.judge(ASSET_RUBRIC, ctx.spec.backends.judge, n_samples=1)
+ judge = ctx.services.judge(ctx.track, ASSET_RUBRIC, ctx.spec.backends.judge, n_samples=1)
except Exception as e: # noqa: BLE001 — asset judging is optional polish
events.emit("asset.judge_unavailable", asset=asset.name, error=str(e))
return result
diff --git a/harness/tests/judges/test_defects.py b/harness/tests/judges/test_defects.py
index e8c24f02..d6fa3c5c 100644
--- a/harness/tests/judges/test_defects.py
+++ b/harness/tests/judges/test_defects.py
@@ -35,19 +35,19 @@ def test_defect_parse_variants():
out = parse_judge_output(_reply(defects=["floating_part"]), R, IDS)
assert out.defects["floating_part"] is True and out.defects["primitive_only"] is False
assert out.defect_evidence["floating_part"] == "MONTAGE 1 top-left"
- # bare booleans and list-of-records are accepted; unknown ids dropped; missing ids → absent
+ # unknown ids dropped; missing ids → absent
rep = good_reply(R, IDS, 0.8)
- rep["defects"] = [{"id": "interpenetration", "present": True}, {"id": "made_up", "present": True}]
+ rep["defects"] = {"interpenetration": {"present": True}, "made_up": {"present": True}}
out2 = parse_judge_output(rep, R, IDS)
assert out2.defects["interpenetration"] is True and "made_up" not in out2.defects
assert out2.defects["floating_part"] is False and out2.defect_evidence["floating_part"] == "not answered by judge"
- rep3 = good_reply(R, IDS, 0.8)
- rep3["defects"] = {"floating_part": True}
- assert parse_judge_output(rep3, R, IDS).defects["floating_part"] is True
rep4 = good_reply(R, IDS, 0.8)
rep4["defects"] = {"floating_part": {"present": "yes"}}
with pytest.raises(JudgeParseError):
parse_judge_output(rep4, R, IDS)
+ rep4["defects"] = [{"id": "floating_part", "present": True}] # not the wire shape: a retry, never an AttributeError
+ with pytest.raises(JudgeParseError):
+ parse_judge_output(rep4, R, IDS)
def test_penalties_and_caps_computed_in_code(judge_input, cache_dir):
@@ -62,7 +62,6 @@ def test_penalties_and_caps_computed_in_code(judge_input, cache_dir):
model2 = FakeChatModel([_reply(0.9, ["floating_part", "primitive_only"])])
j2 = VlmJudge("static_object", chat_model=model2, cache_dir=cache_dir).judge(judge_input)
raw2 = json.loads(j2.raw)
- assert raw2["overall_after_defects"] == pytest.approx(0.72)
assert j2.overall == 0.6 and not j2.passed # capped by defect:floating_part
assert raw2["caps"]["caps_applied"][0]["rule"] == "defect:floating_part"
# the checklist is per-sample data too
@@ -76,9 +75,7 @@ def test_defect_vote_ties_are_absent(judge_input, cache_dir, caplog):
j = VlmJudge("static_object", chat_model=flagged_first, n_samples=2, cache_dir=cache_dir).judge(judge_input)
assert "n_samples=2 is even" in caplog.text
raw = json.loads(j.raw)
- assert raw["defect_votes"]["render_artifacts"] == [True, False]
assert raw["defects"]["render_artifacts"] is False and j.overall == pytest.approx(0.9)
- assert raw["tie_broken"] == ["render_artifacts"]
assert raw["scoring_version"] == 4 and raw["overridden"] == [] # 4: the kind-scoped veto (scene, 2026-10-14)
diff --git a/harness/tests/judges/test_output_schema.py b/harness/tests/judges/test_output_schema.py
index b0ed60e5..acfd0eaa 100644
--- a/harness/tests/judges/test_output_schema.py
+++ b/harness/tests/judges/test_output_schema.py
@@ -11,20 +11,15 @@ R = load_rubric("static_object")
IDS = ["A1", "must-2"]
-def test_parse_accepts_bare_values_defaults_missing_acceptance_and_rejects_bad_criteria():
- rep = good_reply(R, IDS)
- rep["acceptance"] = {"A1": True, "must-2": False}
- rep["criteria"] = {k: v["score"] for k, v in rep["criteria"].items()}
- out = parse_judge_output(rep, R, IDS)
- assert out.acceptance_bools == {"A1": True, "must-2": False}
- assert out.criteria["materials"].evidence == ""
- # a missing acceptance item is unverified; an unknown criterion is dropped
+def test_parse_defaults_missing_acceptance_drops_unknown_ids_and_rejects_bad_criteria():
+ # a missing acceptance item is unverified; an unknown acceptance id or criterion is dropped
rep = good_reply(R, IDS)
+ rep["acceptance"]["zzz"] = {"verified": True, "evidence": ""}
rep["acceptance"].pop("must-2")
rep["criteria"]["bogus"] = {"score": 1.0, "evidence": ""}
out = parse_judge_output(rep, R, IDS)
assert out.acceptance_bools["must-2"] is False
- assert "bogus" not in out.criteria
+ assert "bogus" not in out.criteria and "zzz" not in out.acceptance
# a missing criterion, or a score out of range, is a parse error (the judge retries)
rep = good_reply(R, IDS)
rep["criteria"].pop("materials")
diff --git a/harness/tests/judges/test_vlm_judge.py b/harness/tests/judges/test_vlm_judge.py
index a6c6e0a4..c0d0f62b 100644
--- a/harness/tests/judges/test_vlm_judge.py
+++ b/harness/tests/judges/test_vlm_judge.py
@@ -26,7 +26,7 @@ def test_full_path_single_sample(judge_input, cache_dir, judge_settings):
assert j.judge_backend == judge_settings.backends().judge == "gemini:gemini-4-argon"
assert j.rubric == "static_object"
raw = json.loads(j.raw)
- assert raw["status"] == "ok" and raw["n_used"] == 1 and raw["rubric_hash"] == R.content_hash()
+ assert raw["status"] == "ok" and raw["rubric_hash"] == R.content_hash()
assert "[verdict: overall 0.80 vs threshold 0.72 → PASS]" in j.summary
req = model.requests[0]
assert req.thinking == "medium"
@@ -50,7 +50,7 @@ def test_n_samples_mean_std_and_view_order(judge_input, cache_dir, fixed_order):
j = _judge(model, n_samples=3, cache_dir=cache_dir, fixed_order=fixed_order).judge(judge_input)
assert j.n_samples == 3 and j.overall == pytest.approx(0.8) and j.score_std == pytest.approx(0.0816, abs=1e-3)
assert j.usage.cost_usd == pytest.approx(0.003) and j.usage.input_tokens == 3000
- assert json.loads(j.raw)["per_sample_overall"] == [0.7, 0.9, 0.8]
+ assert len(json.loads(j.raw)["samples"]) == 3
assert j.summary.startswith("A recognisable chair.") # the representative sample's narrative
orders = {tuple(p.label for p in image_parts(r)) for r in model.requests}
assert (len(orders) == 1) if fixed_order else (len(orders) >= 2)
diff --git a/harness/tests/orchestrator_tracks/fakes.py b/harness/tests/orchestrator_tracks/fakes.py
index d097d34b..aba1f505 100644
--- a/harness/tests/orchestrator_tracks/fakes.py
+++ b/harness/tests/orchestrator_tracks/fakes.py
@@ -351,7 +351,7 @@ class FakeServices(Services):
self.motion_errors = motion_errors
self.materialized: list[str] = []
self.records: list[RunRecord] = []
- self.reference_judges: list[tuple[str, int, str]] = []
+ self.judges_made: list[tuple[str, bool]] = []
self.silhouette_calls: list[tuple[str, str]] = []
self.planned_edges: list[list[tuple[str, str]]] = []
@@ -361,11 +361,8 @@ class FakeServices(Services):
def coding_agent(self, agent_id: str) -> Any:
raise ServiceUnavailable("no coding agent in tests")
- def judge(self, rubric: str, model_id: str, n_samples: int = 1) -> Any:
- return self._judge if self._judge is not None else FakeJudge()
-
- def reference_judge(self, model_id: str, n_samples: int = 1, rubric: str = "reference") -> Any:
- self.reference_judges.append((model_id, n_samples, rubric))
+ def judge(self, track: Any, rubric: str, model_id: str, n_samples: int = 1, *, references: bool = False) -> Any:
+ self.judges_made.append((rubric, references))
return self._judge if self._judge is not None else FakeJudge()
def silhouette(self, render_png, reference_png) -> dict[str, Any]:
diff --git a/harness/tests/orchestrator_tracks/test_candidates_reference_motion.py b/harness/tests/orchestrator_tracks/test_candidates_reference_motion.py
index 4c814a4b..e95e1b6d 100644
--- a/harness/tests/orchestrator_tracks/test_candidates_reference_motion.py
+++ b/harness/tests/orchestrator_tracks/test_candidates_reference_motion.py
@@ -115,7 +115,7 @@ def test_reference_images_wire_judge_prompts_and_silhouette(tmp_path, chair_plan
settings=settings, runtime=FakeRuntime(Language.BLENDER))
rec = track.run(spec, ws)
# reference judge + rubric
- assert services.reference_judges and services.reference_judges[0][2] == "reference" and rec.extra["rubric"] == "reference"
+ assert services.judges_made and services.judges_made[0] == ("reference", True) and rec.extra["rubric"] == "reference"
# prompts mention the references; single-shot attaches the image
from codeverse3d.contracts.chat import ImagePart
diff --git a/harness/tests/orchestrator_tracks/test_likeness_routing.py b/harness/tests/orchestrator_tracks/test_likeness_routing.py
index dee6e006..07f57373 100644
--- a/harness/tests/orchestrator_tracks/test_likeness_routing.py
+++ b/harness/tests/orchestrator_tracks/test_likeness_routing.py
@@ -28,19 +28,13 @@ def test_graphics_note_asks_for_the_physics_not_a_silhouette(tmp_path):
assert reference_note(_ctx(Track.GRAPHICS, [str(tmp_path / "gone.png")])) == ""
-def test_make_judge_routes_photos_to_the_likeness_judge_and_a_replay_picks_the_same():
- calls: list[tuple[str, str]] = []
- services = SimpleNamespace(
- likeness_judge=lambda model, n_samples, rubric: calls.append(("likeness", rubric)) or "L",
- reference_judge=lambda model, n_samples, rubric: calls.append(("reference", rubric)) or "R",
- judge=lambda rubric, model, n_samples: calls.append(("plain", rubric)) or "P",
- )
+def test_make_judge_hands_the_track_rubric_and_photos_to_one_factory_and_a_replay_picks_the_same():
+ calls: list[tuple] = []
+ services = SimpleNamespace(judge=lambda track, rubric, model, n_samples, references: calls.append((track, rubric, references)))
me = SimpleNamespace(_judge=None, services=services)
- assert BaseTrack.make_judge(me, _ctx(Track.GRAPHICS, ["/x.png"])) == "L"
- assert BaseTrack.make_judge(me, _ctx(Track.SCENE, ["/x.png"])) == "L"
- assert BaseTrack.make_judge(me, _ctx(Track.STATIC_OBJECT, ["/x.png"])) == "R"
- assert BaseTrack.make_judge(me, _ctx(Track.GRAPHICS, [])) == "P"
- assert calls[0] == ("likeness", "shader")
+ BaseTrack.make_judge(me, _ctx(Track.GRAPHICS, ["/x.png"]))
+ BaseTrack.make_judge(me, _ctx(Track.GRAPHICS, []))
+ assert calls == [(Track.GRAPHICS, "shader", True), (Track.GRAPHICS, "shader", False)]
# a replay picks the judge the live run did: `3dcodeverse judge` / calibration and the live run share
# `judge_for` — a graphics or scene run with photos was re-judged by ReferenceJudge (silhouette +
# diff pass) where the run used likeness
diff --git a/harness/tests/prompts/manifest.json b/harness/tests/prompts/manifest.json
index 11093089..d021fb68 100644
--- a/harness/tests/prompts/manifest.json
+++ b/harness/tests/prompts/manifest.json
@@ -18,11 +18,11 @@
"job:refine@r01/prompt": {"bytes": 7267, "sha256": "a718673dc59d566a5af561d8543756e0326057613430e6dae0cb13eb23b8234e"},
"job:refine@r01/scope": {"bytes": 379, "sha256": "d17ae76997ec9443d52089e32ec81ecac3ba2c93529b227533942e84598113d9"},
"job:refine@r01/system": {"bytes": 3271, "sha256": "29f5a9b2e7b9a879be58963ce95ae6fbec7460dae6785d542b214d6b850e21f4"},
- "judge:articulated:runs/run/artifacts/renders/r00/input": {"bytes": 9819, "sha256": "0a7445d421267800ff75c73621b735d79a7ceb04ab107e640f52c145916b0201"},
+ "judge:articulated:runs/run/artifacts/renders/r00/input": {"bytes": 9774, "sha256": "10640a4a962ca2b86feafecc6e717ae69db6bbd8dd7537060f28ba30cbeebe46"},
"judge:articulated:runs/run/artifacts/renders/r00/messages": {"bytes": 7266, "sha256": "44869437393997f2107fdf0a59b70d830557064fa5958abfbee786e535323a2c"},
"judge:articulated:runs/run/artifacts/renders/r00/sampling": {"bytes": 11992, "sha256": "0450937509be0de6976f74c7d439af0878caa67d4105d01afcec4a211cfbfbd2"},
"judge:articulated:runs/run/artifacts/renders/r00/system": {"bytes": 13129, "sha256": "b6c7c917b035224ca53b1a3e1fcd0c838b8ac20563e687cd6521d8cab4a78ed6"},
- "judge:articulated:runs/run/artifacts/renders/r01/input": {"bytes": 10722, "sha256": "67b607398aa3997504db43be1092ecec063fc228f55af02dc66755ee602b993a"},
+ "judge:articulated:runs/run/artifacts/renders/r01/input": {"bytes": 10677, "sha256": "208dcdbbb369608c39b62827783f2ebddb74b7df32fb5932e5dda66327a041f2"},
"judge:articulated:runs/run/artifacts/renders/r01/messages": {"bytes": 7591, "sha256": "a2e7394b1d8515b0256e914ceb6b07eaf526a634104ea03829c559e5fd7ebf67"},
"judge:articulated:runs/run/artifacts/renders/r01/sampling": {"bytes": 11992, "sha256": "0450937509be0de6976f74c7d439af0878caa67d4105d01afcec4a211cfbfbd2"},
"judge:articulated:runs/run/artifacts/renders/r01/system": {"bytes": 13129, "sha256": "b6c7c917b035224ca53b1a3e1fcd0c838b8ac20563e687cd6521d8cab4a78ed6"},
@@ -51,11 +51,11 @@
"chat:planner:planner/messages": {"bytes": 1328, "sha256": "313c87553cf38df423dc9245a8af4cca5a41a844b489e13a5dc08e8c44eabba5"},
"chat:planner:planner/sampling": {"bytes": 10001, "sha256": "0ff37a3ae43fdf878dd0911a9954d57e49365ad788c84818ea01d350310ab401"},
"chat:planner:planner/system": {"bytes": 20030, "sha256": "45dccfcf72b4c6e46b7ceeaae8af0ebddf6f6e6b5970cda5fe900aa47682bf06"},
- "judge:articulated:runs/run/artifacts/renders/r00/input": {"bytes": 9539, "sha256": "f0513f5660eed412e9e260200d50ba2cb4d5ea144d252fe7b6a86a4c94b0306d"},
+ "judge:articulated:runs/run/artifacts/renders/r00/input": {"bytes": 9494, "sha256": "6979f3d46aa69e74e061de59815a7686cb7adb6410ad365b1381d465fb91a17a"},
"judge:articulated:runs/run/artifacts/renders/r00/messages": {"bytes": 7120, "sha256": "7a0ef8d51456faa2c38867eea6c502fbe1a7aa30d6cd9f23b44701a2d11846bf"},
"judge:articulated:runs/run/artifacts/renders/r00/sampling": {"bytes": 11992, "sha256": "0450937509be0de6976f74c7d439af0878caa67d4105d01afcec4a211cfbfbd2"},
"judge:articulated:runs/run/artifacts/renders/r00/system": {"bytes": 13129, "sha256": "b6c7c917b035224ca53b1a3e1fcd0c838b8ac20563e687cd6521d8cab4a78ed6"},
- "judge:articulated:runs/run/artifacts/renders/r01/input": {"bytes": 10442, "sha256": "faeb5630c2e4ba0081b5ec45b166279deb1ad38548014ef829110a2c452d8d3d"},
+ "judge:articulated:runs/run/artifacts/renders/r01/input": {"bytes": 10397, "sha256": "47f43a066f59487f10617337e9926281444ab21d475cbec139ff356f1b8a1796"},
"judge:articulated:runs/run/artifacts/renders/r01/messages": {"bytes": 7445, "sha256": "2a654e5bb10f8a642367b40c8893a16dce24c4bc4f6941cfbba76ce348ad4f1c"},
"judge:articulated:runs/run/artifacts/renders/r01/sampling": {"bytes": 11992, "sha256": "0450937509be0de6976f74c7d439af0878caa67d4105d01afcec4a211cfbfbd2"},
"judge:articulated:runs/run/artifacts/renders/r01/system": {"bytes": 13129, "sha256": "b6c7c917b035224ca53b1a3e1fcd0c838b8ac20563e687cd6521d8cab4a78ed6"},
@@ -81,15 +81,15 @@
"job:baseline_c1@r00/prompt": {"bytes": 11347, "sha256": "5aca38cfae577e7e5e5974c2b6cd64e55ec35a1dfab33eaf62eff70652854df5"},
"job:baseline_c1@r00/scope": {"bytes": 383, "sha256": "831265b54627f1f6c1bfadebefa4e898d6b543b253bf6be0685b5383bd39fb57"},
"job:baseline_c1@r00/system": {"bytes": 3173, "sha256": "c3879a563e2a512b82358d0c3927dcbfe9347725e592035a9b64c5d1b34e6654"},
- "judge:shader:runs/run/_cand/c0/artifacts/renders/r00/input": {"bytes": 4013, "sha256": "53102aacf0ecbb26bb489751b12f64205ca575c6ed6005650f9297108f4e2577"},
+ "judge:shader:runs/run/_cand/c0/artifacts/renders/r00/input": {"bytes": 3994, "sha256": "9017ce25c58c838de8f1d4b5b06b1e229231815e0a1be8423fe1ad59fcafd350"},
"judge:shader:runs/run/_cand/c0/artifacts/renders/r00/messages": {"bytes": 3727, "sha256": "d7deb2c4a1fa8e9b015a4e81439f9781e7618be6a01b013d18bb3acc4f6a32b9"},
"judge:shader:runs/run/_cand/c0/artifacts/renders/r00/sampling": {"bytes": 11662, "sha256": "88d8e8ec41fe9fc983cb78208877ac3d1c9148a4f67637ee61b6e23451a9e0dc"},
"judge:shader:runs/run/_cand/c0/artifacts/renders/r00/system": {"bytes": 11253, "sha256": "dcf75b56942831a1db5074023dfdad4b4750a3842eb4233dc2d41adacd1de67b"},
- "judge:shader:runs/run/_cand/c1/artifacts/renders/r00/input": {"bytes": 4013, "sha256": "0cd59e323b50bd972a8187ab1752601a68a09be5b4cad5a8ff31272358edb36a"},
+ "judge:shader:runs/run/_cand/c1/artifacts/renders/r00/input": {"bytes": 3994, "sha256": "33e92e986d31784b0cfabb948d5ec11f370b2648fb583e248920e955d0df6934"},
"judge:shader:runs/run/_cand/c1/artifacts/renders/r00/messages": {"bytes": 3727, "sha256": "d7deb2c4a1fa8e9b015a4e81439f9781e7618be6a01b013d18bb3acc4f6a32b9"},
"judge:shader:runs/run/_cand/c1/artifacts/renders/r00/sampling": {"bytes": 11662, "sha256": "88d8e8ec41fe9fc983cb78208877ac3d1c9148a4f67637ee61b6e23451a9e0dc"},
"judge:shader:runs/run/_cand/c1/artifacts/renders/r00/system": {"bytes": 11253, "sha256": "dcf75b56942831a1db5074023dfdad4b4750a3842eb4233dc2d41adacd1de67b"},
- "judge:shader:runs/run/artifacts/renders/r00/input": {"bytes": 3959, "sha256": "5a6cd939181903bcd28e49d83b7f86b407558242c1e86a91c82bd4fb147dd0ef"},
+ "judge:shader:runs/run/artifacts/renders/r00/input": {"bytes": 3940, "sha256": "5b42c0e9bf4425488c783ed454fa0f483dec9c992d931089f5f0455aa698d0b1"},
"judge:shader:runs/run/artifacts/renders/r00/messages": {"bytes": 3727, "sha256": "d7deb2c4a1fa8e9b015a4e81439f9781e7618be6a01b013d18bb3acc4f6a32b9"},
"judge:shader:runs/run/artifacts/renders/r00/sampling": {"bytes": 11662, "sha256": "88d8e8ec41fe9fc983cb78208877ac3d1c9148a4f67637ee61b6e23451a9e0dc"},
"judge:shader:runs/run/artifacts/renders/r00/system": {"bytes": 11253, "sha256": "dcf75b56942831a1db5074023dfdad4b4750a3842eb4233dc2d41adacd1de67b"},
@@ -117,15 +117,15 @@
"job:refine@r02/prompt": {"bytes": 4828, "sha256": "f02f3a218edb95d79000fd339cc556045baccd5087231dd293db83195c927fa7"},
"job:refine@r02/scope": {"bytes": 375, "sha256": "c4a7fdb0b390ec75d595006e11b9982113e60c49f5a62a978153167053bbcd33"},
"job:refine@r02/system": {"bytes": 3173, "sha256": "c3879a563e2a512b82358d0c3927dcbfe9347725e592035a9b64c5d1b34e6654"},
- "judge:shader:runs/run/artifacts/renders/r00/input": {"bytes": 3959, "sha256": "4dde2639e506ef0d2636bde47835d14b7f8702f43bdf5f4d7c6072ed65c6637c"},
+ "judge:shader:runs/run/artifacts/renders/r00/input": {"bytes": 3940, "sha256": "76bcc0b648da93801f0388ce5f7fa927bbaef4294992a4f97d61810d873fde02"},
"judge:shader:runs/run/artifacts/renders/r00/messages": {"bytes": 3727, "sha256": "d7deb2c4a1fa8e9b015a4e81439f9781e7618be6a01b013d18bb3acc4f6a32b9"},
"judge:shader:runs/run/artifacts/renders/r00/sampling": {"bytes": 11662, "sha256": "88d8e8ec41fe9fc983cb78208877ac3d1c9148a4f67637ee61b6e23451a9e0dc"},
"judge:shader:runs/run/artifacts/renders/r00/system": {"bytes": 11253, "sha256": "dcf75b56942831a1db5074023dfdad4b4750a3842eb4233dc2d41adacd1de67b"},
- "judge:shader:runs/run/artifacts/renders/r01/input": {"bytes": 5012, "sha256": "2eadea804479eca53e39ff1fb2b505e71188e16cb38241b7e77dce40d5f97032"},
+ "judge:shader:runs/run/artifacts/renders/r01/input": {"bytes": 4993, "sha256": "6cfdce1740f3e23ee5dacff9505abc54ff248e606d94ca9af0b43cdc490952b0"},
"judge:shader:runs/run/artifacts/renders/r01/messages": {"bytes": 4123, "sha256": "a1f0c207406de6f415fdb8516b695aaedf7f06d25598f33932d2ad728f8c881f"},
"judge:shader:runs/run/artifacts/renders/r01/sampling": {"bytes": 11662, "sha256": "88d8e8ec41fe9fc983cb78208877ac3d1c9148a4f67637ee61b6e23451a9e0dc"},
"judge:shader:runs/run/artifacts/renders/r01/system": {"bytes": 11253, "sha256": "dcf75b56942831a1db5074023dfdad4b4750a3842eb4233dc2d41adacd1de67b"},
- "judge:shader:runs/run/artifacts/renders/r02/input": {"bytes": 5008, "sha256": "260f183b612bd9ae84a5ad913e2fdd84788d144f9a24a14aba70d410caf861f6"},
+ "judge:shader:runs/run/artifacts/renders/r02/input": {"bytes": 4989, "sha256": "3f80efe4c05ef89f372e24d4a15bac9d8168ae9e4009a0a989e8d116ec5d68ab"},
"judge:shader:runs/run/artifacts/renders/r02/messages": {"bytes": 4123, "sha256": "5b81b09f9d6a7e3225549454099d07143f5bb726c3ccff263b4afc7bfe13d88b"},
"judge:shader:runs/run/artifacts/renders/r02/sampling": {"bytes": 11662, "sha256": "88d8e8ec41fe9fc983cb78208877ac3d1c9148a4f67637ee61b6e23451a9e0dc"},
"judge:shader:runs/run/artifacts/renders/r02/system": {"bytes": 11253, "sha256": "dcf75b56942831a1db5074023dfdad4b4750a3842eb4233dc2d41adacd1de67b"},
@@ -156,7 +156,7 @@
"job:rebuild@r01/prompt": {"bytes": 11530, "sha256": "55b953a2ecd60c415cec8995694d1dc731bd0d8135bc594a1ba07d68b7f5f09f"},
"job:rebuild@r01/scope": {"bytes": 379, "sha256": "471fba995682e47fe12099cb5881743a8aeae71a0243fa3598c72d69a5b31f98"},
"job:rebuild@r01/system": {"bytes": 3173, "sha256": "c3879a563e2a512b82358d0c3927dcbfe9347725e592035a9b64c5d1b34e6654"},
- "judge:shader:runs/run/artifacts/renders/r01/input": {"bytes": 3959, "sha256": "df77c1b050ec2b6aacb408eeb09dc6b9ab0bd37e6aadea89a54db3ce9dffede3"},
+ "judge:shader:runs/run/artifacts/renders/r01/input": {"bytes": 3940, "sha256": "e6cc828e4e29dcf2648d0e1d6f64786b693275e0c29dd029228967f047ae75f7"},
"judge:shader:runs/run/artifacts/renders/r01/messages": {"bytes": 3727, "sha256": "d7deb2c4a1fa8e9b015a4e81439f9781e7618be6a01b013d18bb3acc4f6a32b9"},
"judge:shader:runs/run/artifacts/renders/r01/sampling": {"bytes": 11662, "sha256": "88d8e8ec41fe9fc983cb78208877ac3d1c9148a4f67637ee61b6e23451a9e0dc"},
"judge:shader:runs/run/artifacts/renders/r01/system": {"bytes": 11253, "sha256": "dcf75b56942831a1db5074023dfdad4b4750a3842eb4233dc2d41adacd1de67b"},
@@ -179,11 +179,11 @@
"chat:planner:planner/messages": {"bytes": 348, "sha256": "f50fc3cdf3a426fc9c9b830b6e68d2cfe9fd9b7125550362b018f9c9b454b25c"},
"chat:planner:planner/sampling": {"bytes": 3421, "sha256": "918170272cea656e8c542c54b777c57d1a4567a8cc36a0c1cd825cb81a01b9bb"},
"chat:planner:planner/system": {"bytes": 11919, "sha256": "194d0e2a724c4799816033228af15cac568dd51c14ed85bff9c94ae5ba281866"},
- "judge:shader:runs/run/artifacts/renders/r00/input": {"bytes": 3976, "sha256": "6047e9f7c3985e1827f3ecc3263d87012c8a3c074b9bc33517e11e12751d2e58"},
+ "judge:shader:runs/run/artifacts/renders/r00/input": {"bytes": 3957, "sha256": "0329b91e396067d845a7fccd9fe2bb021d62d8b74e38e2b3f1554c6d439c5acf"},
"judge:shader:runs/run/artifacts/renders/r00/messages": {"bytes": 3727, "sha256": "d7deb2c4a1fa8e9b015a4e81439f9781e7618be6a01b013d18bb3acc4f6a32b9"},
"judge:shader:runs/run/artifacts/renders/r00/sampling": {"bytes": 11662, "sha256": "88d8e8ec41fe9fc983cb78208877ac3d1c9148a4f67637ee61b6e23451a9e0dc"},
"judge:shader:runs/run/artifacts/renders/r00/system": {"bytes": 11253, "sha256": "dcf75b56942831a1db5074023dfdad4b4750a3842eb4233dc2d41adacd1de67b"},
- "judge:shader:runs/run/artifacts/renders/r01/input": {"bytes": 5029, "sha256": "d3e2b3031ba4e6593d3c55af498869db17277952dcf5f320ac415bc04616aca7"},
+ "judge:shader:runs/run/artifacts/renders/r01/input": {"bytes": 5010, "sha256": "d3f2975d0a9c9b79bd0902e39a6509b124e53effa66d9f1bd542feebd38208cf"},
"judge:shader:runs/run/artifacts/renders/r01/messages": {"bytes": 4123, "sha256": "a1f0c207406de6f415fdb8516b695aaedf7f06d25598f33932d2ad728f8c881f"},
"judge:shader:runs/run/artifacts/renders/r01/sampling": {"bytes": 11662, "sha256": "88d8e8ec41fe9fc983cb78208877ac3d1c9148a4f67637ee61b6e23451a9e0dc"},
"judge:shader:runs/run/artifacts/renders/r01/system": {"bytes": 11253, "sha256": "dcf75b56942831a1db5074023dfdad4b4750a3842eb4233dc2d41adacd1de67b"},
@@ -207,11 +207,11 @@
"job:refine@r01/prompt": {"bytes": 4809, "sha256": "564c35401c05213d9906d0ecf84ead9e8c8a54ef51cae13eeb59fcb98e1b7275"},
"job:refine@r01/scope": {"bytes": 354, "sha256": "81a4042ba077832530b2a9f64d3e1415da1b655b00caf1006ed51ee616a7b17b"},
"job:refine@r01/system": {"bytes": 2729, "sha256": "d12d4891ce72156aa561bdfa0ed1fb20c5826412024b0be07b423949528e5df6"},
- "judge:shader:runs/run/artifacts/renders/r00/input": {"bytes": 3963, "sha256": "5401be1bc487921af05b2dc05e8dc7c0fb7f2ee373d2741928c54417eee99b6e"},
+ "judge:shader:runs/run/artifacts/renders/r00/input": {"bytes": 3944, "sha256": "67ad3296a96af7a97e97593c3b322aa52f28b84d19c55ea53890a10e65310b8c"},
"judge:shader:runs/run/artifacts/renders/r00/messages": {"bytes": 3731, "sha256": "355b5a5eb7617171897155fe818a3975dea97d127ae8416f1352a67bb6a84a19"},
"judge:shader:runs/run/artifacts/renders/r00/sampling": {"bytes": 11662, "sha256": "88d8e8ec41fe9fc983cb78208877ac3d1c9148a4f67637ee61b6e23451a9e0dc"},
"judge:shader:runs/run/artifacts/renders/r00/system": {"bytes": 11253, "sha256": "dcf75b56942831a1db5074023dfdad4b4750a3842eb4233dc2d41adacd1de67b"},
- "judge:shader:runs/run/artifacts/renders/r01/input": {"bytes": 5016, "sha256": "6e97c9effd603ac4a768006e2b4c63ce29506fe163aae8c36087f3f2f62390fc"},
+ "judge:shader:runs/run/artifacts/renders/r01/input": {"bytes": 4997, "sha256": "ec17ede71c6393b5b990c777e25bbce8d5078f08ae1d9d12f74fec13d28e35f6"},
"judge:shader:runs/run/artifacts/renders/r01/messages": {"bytes": 4127, "sha256": "a4c4a41eea8d4af3df6646d2aa757c01a0ab663614361e61e76e84e21f9d5378"},
"judge:shader:runs/run/artifacts/renders/r01/sampling": {"bytes": 11662, "sha256": "88d8e8ec41fe9fc983cb78208877ac3d1c9148a4f67637ee61b6e23451a9e0dc"},
"judge:shader:runs/run/artifacts/renders/r01/system": {"bytes": 11253, "sha256": "dcf75b56942831a1db5074023dfdad4b4750a3842eb4233dc2d41adacd1de67b"},
@@ -281,15 +281,15 @@
"job:zones_quay_water@r00/prompt": {"bytes": 35057, "sha256": "8e18d1a7baa4f7747fabe9affb187b29ba5a39e9e2d5342ebe5027c9ec20c68c"},
"job:zones_quay_water@r00/scope": {"bytes": 364, "sha256": "69a91221629a23e4f5bc8ce393294d5c6a00213156b2d2ded4783159a0f02557"},
"job:zones_quay_water@r00/system": {"bytes": 7239, "sha256": "67334373ec84ee3a89a3870646aef6a41f03b9ae15678e06e800bc206e10673d"},
- "judge:scene:runs/run/artifacts/renders/r00/input": {"bytes": 3690, "sha256": "8b1b52e0e3fd832fda14a7913d35cef8d5bda2cb02b8aea5a2935bbc1f7d5714"},
+ "judge:scene:runs/run/artifacts/renders/r00/input": {"bytes": 3671, "sha256": "d9ebd5b9a5d5c66c8f71f019376d002a32259926864e2e10766cb54486e94cbc"},
"judge:scene:runs/run/artifacts/renders/r00/messages": {"bytes": 4930, "sha256": "676859b5172ea33c86195b543b5c80e49dc907598dbdddb57a9734e58df9e8a1"},
"judge:scene:runs/run/artifacts/renders/r00/sampling": {"bytes": 14069, "sha256": "9f5073025f0011d617d13bbde6605bf7358111b7700df92c41c23035482f789b"},
"judge:scene:runs/run/artifacts/renders/r00/system": {"bytes": 21572, "sha256": "eee5e34a58a91720755be68ed4d39cf127c634a484a4948e77c77ecce7e8cd16"},
- "judge:scene:runs/run/artifacts/renders/r01/input": {"bytes": 4690, "sha256": "b546f93804fc3a29f245076f24cee36e64a1a7a5d45181f05eacc5bf59fc2dc0"},
+ "judge:scene:runs/run/artifacts/renders/r01/input": {"bytes": 4671, "sha256": "85ff73f1331745397559c9d5a8acbb3e440e1597cb6e003b2f7643d26da9386c"},
"judge:scene:runs/run/artifacts/renders/r01/messages": {"bytes": 5298, "sha256": "35679341c294b774ac1c228d9a2bc56f4792f8831dd1c216b513e4427e6cb0c1"},
"judge:scene:runs/run/artifacts/renders/r01/sampling": {"bytes": 14069, "sha256": "9f5073025f0011d617d13bbde6605bf7358111b7700df92c41c23035482f789b"},
"judge:scene:runs/run/artifacts/renders/r01/system": {"bytes": 21572, "sha256": "eee5e34a58a91720755be68ed4d39cf127c634a484a4948e77c77ecce7e8cd16"},
- "judge:scene:runs/run/artifacts/renders/r02/input": {"bytes": 4686, "sha256": "5321b87318ce2cbdd7d3b6d6669d3283574a1e3034030803577f386c6e0d2d35"},
+ "judge:scene:runs/run/artifacts/renders/r02/input": {"bytes": 4667, "sha256": "8d84413ef22fb9bc007c85bd7dd3d797dfad27001dcf4e189d15bf788948699e"},
"judge:scene:runs/run/artifacts/renders/r02/messages": {"bytes": 5298, "sha256": "0b15d27d13b64c082c86c2e4cea5d85a7f0b3e9daef93b5cb62f698d06570437"},
"judge:scene:runs/run/artifacts/renders/r02/sampling": {"bytes": 14069, "sha256": "9f5073025f0011d617d13bbde6605bf7358111b7700df92c41c23035482f789b"},
"judge:scene:runs/run/artifacts/renders/r02/system": {"bytes": 21572, "sha256": "eee5e34a58a91720755be68ed4d39cf127c634a484a4948e77c77ecce7e8cd16"},
@@ -347,7 +347,7 @@
"chat:planner:zone-layout#2/messages": {"bytes": 485, "sha256": "67e917190ee227c5ed3415ab4b03c072af313d7040704e7de671e63c42e2ab52"},
"chat:planner:zone-layout#2/sampling": {"bytes": 3397, "sha256": "160445a83ede28afc0191547fd42d9341e694ef70aaad380bebf60ead0a65c49"},
"chat:planner:zone-layout#2/system": {"bytes": 3326, "sha256": "494acec302dd23fa2fc6ee1eb6d48e873d6cf0ad98bad413e27578cf062313d4"},
- "judge:scene:runs/run/artifacts/renders/r00/input": {"bytes": 3707, "sha256": "30b3e2a3c085d3231818fcaa188e1f93732151dd987f311121f2a5320110c27d"},
+ "judge:scene:runs/run/artifacts/renders/r00/input": {"bytes": 3688, "sha256": "f4a2f469cd1a57547c5edd0691db35c4f75ae4b5a8cded221bdef01f323c3b6f"},
"judge:scene:runs/run/artifacts/renders/r00/messages": {"bytes": 4930, "sha256": "676859b5172ea33c86195b543b5c80e49dc907598dbdddb57a9734e58df9e8a1"},
"judge:scene:runs/run/artifacts/renders/r00/sampling": {"bytes": 14069, "sha256": "9f5073025f0011d617d13bbde6605bf7358111b7700df92c41c23035482f789b"},
"judge:scene:runs/run/artifacts/renders/r00/system": {"bytes": 21572, "sha256": "eee5e34a58a91720755be68ed4d39cf127c634a484a4948e77c77ecce7e8cd16"},
@@ -411,15 +411,15 @@
"job:zones_quay_water@r00/prompt": {"bytes": 42361, "sha256": "ba7f066f892074d3b6230f7380a1a3492167ab8c24a045e3ee38f0acab6a420a"},
"job:zones_quay_water@r00/scope": {"bytes": 379, "sha256": "5b1def419a1370ab2d9f2345bc4a5232ef3c793d4e4f1e01bd5ab9a3a8e88a7d"},
"job:zones_quay_water@r00/system": {"bytes": 5625, "sha256": "3e4c806c832c2fb4ab9dd3b1d5063c611b07258ce2fa6d9b3f25df5a467734c7"},
- "judge:scene:runs/run/artifacts/renders/r00/input": {"bytes": 3690, "sha256": "5e7031f1c8d724681399e545e6adf0af8e6a8da6cae4364c7220a15b41f0744b"},
+ "judge:scene:runs/run/artifacts/renders/r00/input": {"bytes": 3671, "sha256": "e0a279e07356ac991389dd13a46ffa63925c422c124e9a529b68a698c03d51dd"},
"judge:scene:runs/run/artifacts/renders/r00/messages": {"bytes": 4930, "sha256": "562ef4bfc6b24c34604482cb73a02fb4b01da5c0e70685e3a1a4e1dda616a827"},
"judge:scene:runs/run/artifacts/renders/r00/sampling": {"bytes": 14069, "sha256": "9f5073025f0011d617d13bbde6605bf7358111b7700df92c41c23035482f789b"},
"judge:scene:runs/run/artifacts/renders/r00/system": {"bytes": 21572, "sha256": "eee5e34a58a91720755be68ed4d39cf127c634a484a4948e77c77ecce7e8cd16"},
- "judge:scene:runs/run/artifacts/renders/r01/input": {"bytes": 4690, "sha256": "ea728103072ad97014d8af107bb40109b9aef55ab30f817a017e3542b091682a"},
+ "judge:scene:runs/run/artifacts/renders/r01/input": {"bytes": 4671, "sha256": "b229d374a8e6f6bc73dc6ecbc0b5ea6e3d512e58f6417b8fb4fe147e8e795dcc"},
"judge:scene:runs/run/artifacts/renders/r01/messages": {"bytes": 5298, "sha256": "c9a488e3385bb803ce084eda9a1ad2ed58c338201f1ad6d7f416cf0070f1a9f1"},
"judge:scene:runs/run/artifacts/renders/r01/sampling": {"bytes": 14069, "sha256": "9f5073025f0011d617d13bbde6605bf7358111b7700df92c41c23035482f789b"},
"judge:scene:runs/run/artifacts/renders/r01/system": {"bytes": 21572, "sha256": "eee5e34a58a91720755be68ed4d39cf127c634a484a4948e77c77ecce7e8cd16"},
- "judge:scene:runs/run/artifacts/renders/r02/input": {"bytes": 4686, "sha256": "905a0f9912bb1052573d52017e5f838a18d429827cbcd52be7e6a7f6f40842f0"},
+ "judge:scene:runs/run/artifacts/renders/r02/input": {"bytes": 4667, "sha256": "3891bd56dfcc04f73f8934501c81af1c02381687e94410341e6ab55f6a9426e3"},
"judge:scene:runs/run/artifacts/renders/r02/messages": {"bytes": 5298, "sha256": "d8e7af12d299bcc4d09f7b68a6cdf3de2816e89d0d785ff88153c9e3aa5ce8cd"},
"judge:scene:runs/run/artifacts/renders/r02/sampling": {"bytes": 14069, "sha256": "9f5073025f0011d617d13bbde6605bf7358111b7700df92c41c23035482f789b"},
"judge:scene:runs/run/artifacts/renders/r02/system": {"bytes": 21572, "sha256": "eee5e34a58a91720755be68ed4d39cf127c634a484a4948e77c77ecce7e8cd16"},
@@ -485,15 +485,15 @@
"job:zones_quay_water@r00/prompt": {"bytes": 42541, "sha256": "986c0f46f90e4f652c336741d6925c1bf7f7b2d83dbe5977ba8a9856dfa4c576"},
"job:zones_quay_water@r00/scope": {"bytes": 379, "sha256": "5b1def419a1370ab2d9f2345bc4a5232ef3c793d4e4f1e01bd5ab9a3a8e88a7d"},
"job:zones_quay_water@r00/system": {"bytes": 5625, "sha256": "3e4c806c832c2fb4ab9dd3b1d5063c611b07258ce2fa6d9b3f25df5a467734c7"},
- "judge:asset:runs/run/_assets/crate/artifacts/renders/r00/assets/crate/input": {"bytes": 3759, "sha256": "d3507d23f5a84d828cef7b6b6f9b2c5df3795a2c300ee7aaeb038af59267fc30"},
+ "judge:asset:runs/run/_assets/crate/artifacts/renders/r00/assets/crate/input": {"bytes": 3740, "sha256": "7ed1c4ad9bbe46627388ef6456e548a7bcea7a3cf2d81bd411a6661490c45d84"},
"judge:asset:runs/run/_assets/crate/artifacts/renders/r00/assets/crate/messages": {"bytes": 3099, "sha256": "eee511c765a105054417cb3fa37bf67c24bcf9022da44dc086385878a61d5f67"},
"judge:asset:runs/run/_assets/crate/artifacts/renders/r00/assets/crate/sampling": {"bytes": 8358, "sha256": "d816200df3d61ed4afb3251779d3981fe0bdf6ef70db86cbba3dd76dcc28edf6"},
"judge:asset:runs/run/_assets/crate/artifacts/renders/r00/assets/crate/system": {"bytes": 8834, "sha256": "8b52aa8a0e80ff6ba98f494c36013745f5dd07a4e1040d72ff7f55aeeb907ba5"},
- "judge:asset:runs/run/_assets/crate/artifacts/renders/r00/assets/crate_fix/input": {"bytes": 3779, "sha256": "91ec992fade8a7c1907c0888c2a65715a0ab1ec40ed8d2cd1037a976cf6e82e8"},
+ "judge:asset:runs/run/_assets/crate/artifacts/renders/r00/assets/crate_fix/input": {"bytes": 3760, "sha256": "db0cd0f5265e7b6242276d4c247285af8cc63b7968663ab1d756b0f6e39a3ff1"},
"judge:asset:runs/run/_assets/crate/artifacts/renders/r00/assets/crate_fix/messages": {"bytes": 3099, "sha256": "eee511c765a105054417cb3fa37bf67c24bcf9022da44dc086385878a61d5f67"},
"judge:asset:runs/run/_assets/crate/artifacts/renders/r00/assets/crate_fix/sampling": {"bytes": 8358, "sha256": "d816200df3d61ed4afb3251779d3981fe0bdf6ef70db86cbba3dd76dcc28edf6"},
"judge:asset:runs/run/_assets/crate/artifacts/renders/r00/assets/crate_fix/system": {"bytes": 8834, "sha256": "8b52aa8a0e80ff6ba98f494c36013745f5dd07a4e1040d72ff7f55aeeb907ba5"},
- "judge:scene:runs/run/artifacts/renders/r00/input": {"bytes": 3697, "sha256": "dab5e24ca050491a18ba9f81e894fb7f1103e9d3a16bf93c708ac7fd6ab54944"},
+ "judge:scene:runs/run/artifacts/renders/r00/input": {"bytes": 3678, "sha256": "c568be390c9629408785c20ba5d786b9df369f0d446a67258a4723be082d0a3f"},
"judge:scene:runs/run/artifacts/renders/r00/messages": {"bytes": 4937, "sha256": "407f918497c4e96ed25d4a9c5c968993644c2a43fadfcc1bc6a0c4f7e6a9d99f"},
"judge:scene:runs/run/artifacts/renders/r00/sampling": {"bytes": 14069, "sha256": "9f5073025f0011d617d13bbde6605bf7358111b7700df92c41c23035482f789b"},
"judge:scene:runs/run/artifacts/renders/r00/system": {"bytes": 21572, "sha256": "eee5e34a58a91720755be68ed4d39cf127c634a484a4948e77c77ecce7e8cd16"},
@@ -529,7 +529,7 @@
"job:zones_quay_water@r00/prompt": {"bytes": 41688, "sha256": "752f5e1a1ee0e088ac14df293d7d28d56a29c6d8249f3a2a13f1d1e2249964df"},
"job:zones_quay_water@r00/scope": {"bytes": 379, "sha256": "5b1def419a1370ab2d9f2345bc4a5232ef3c793d4e4f1e01bd5ab9a3a8e88a7d"},
"job:zones_quay_water@r00/system": {"bytes": 5625, "sha256": "3e4c806c832c2fb4ab9dd3b1d5063c611b07258ce2fa6d9b3f25df5a467734c7"},
- "judge:scene:runs/run/artifacts/renders/r00/input": {"bytes": 3690, "sha256": "ebaaf972b6033176defee1013c1eb8b83843caef8172efe2dba99ccfb339a33f"},
+ "judge:scene:runs/run/artifacts/renders/r00/input": {"bytes": 3671, "sha256": "8db21e497c7c1a78034468aed62ee16133c954517a230a1a8fd1a3bc32f4ac34"},
"judge:scene:runs/run/artifacts/renders/r00/messages": {"bytes": 4930, "sha256": "562ef4bfc6b24c34604482cb73a02fb4b01da5c0e70685e3a1a4e1dda616a827"},
"judge:scene:runs/run/artifacts/renders/r00/sampling": {"bytes": 14069, "sha256": "9f5073025f0011d617d13bbde6605bf7358111b7700df92c41c23035482f789b"},
"judge:scene:runs/run/artifacts/renders/r00/system": {"bytes": 21572, "sha256": "eee5e34a58a91720755be68ed4d39cf127c634a484a4948e77c77ecce7e8cd16"},
@@ -636,11 +636,11 @@
"job:zones_quay_water@r00/prompt": {"bytes": 42361, "sha256": "ba7f066f892074d3b6230f7380a1a3492167ab8c24a045e3ee38f0acab6a420a"},
"job:zones_quay_water@r00/scope": {"bytes": 379, "sha256": "5b1def419a1370ab2d9f2345bc4a5232ef3c793d4e4f1e01bd5ab9a3a8e88a7d"},
"job:zones_quay_water@r00/system": {"bytes": 5625, "sha256": "3e4c806c832c2fb4ab9dd3b1d5063c611b07258ce2fa6d9b3f25df5a467734c7"},
- "judge:scene:runs/run/artifacts/renders/r00/input": {"bytes": 3690, "sha256": "5e7031f1c8d724681399e545e6adf0af8e6a8da6cae4364c7220a15b41f0744b"},
+ "judge:scene:runs/run/artifacts/renders/r00/input": {"bytes": 3671, "sha256": "e0a279e07356ac991389dd13a46ffa63925c422c124e9a529b68a698c03d51dd"},
"judge:scene:runs/run/artifacts/renders/r00/messages": {"bytes": 4930, "sha256": "562ef4bfc6b24c34604482cb73a02fb4b01da5c0e70685e3a1a4e1dda616a827"},
"judge:scene:runs/run/artifacts/renders/r00/sampling": {"bytes": 14069, "sha256": "9f5073025f0011d617d13bbde6605bf7358111b7700df92c41c23035482f789b"},
"judge:scene:runs/run/artifacts/renders/r00/system": {"bytes": 21572, "sha256": "eee5e34a58a91720755be68ed4d39cf127c634a484a4948e77c77ecce7e8cd16"},
- "judge:scene:runs/run/artifacts/renders/r01/input": {"bytes": 4690, "sha256": "ea728103072ad97014d8af107bb40109b9aef55ab30f817a017e3542b091682a"},
+ "judge:scene:runs/run/artifacts/renders/r01/input": {"bytes": 4671, "sha256": "b229d374a8e6f6bc73dc6ecbc0b5ea6e3d512e58f6417b8fb4fe147e8e795dcc"},
"judge:scene:runs/run/artifacts/renders/r01/messages": {"bytes": 5298, "sha256": "c9a488e3385bb803ce084eda9a1ad2ed58c338201f1ad6d7f416cf0070f1a9f1"},
"judge:scene:runs/run/artifacts/renders/r01/sampling": {"bytes": 14069, "sha256": "9f5073025f0011d617d13bbde6605bf7358111b7700df92c41c23035482f789b"},
"judge:scene:runs/run/artifacts/renders/r01/system": {"bytes": 21572, "sha256": "eee5e34a58a91720755be68ed4d39cf127c634a484a4948e77c77ecce7e8cd16"},
@@ -697,7 +697,7 @@
"chat:planner:zone-layout#2/messages": {"bytes": 485, "sha256": "67e917190ee227c5ed3415ab4b03c072af313d7040704e7de671e63c42e2ab52"},
"chat:planner:zone-layout#2/sampling": {"bytes": 3397, "sha256": "160445a83ede28afc0191547fd42d9341e694ef70aaad380bebf60ead0a65c49"},
"chat:planner:zone-layout#2/system": {"bytes": 3326, "sha256": "494acec302dd23fa2fc6ee1eb6d48e873d6cf0ad98bad413e27578cf062313d4"},
- "judge:scene:runs/run/artifacts/renders/r00/input": {"bytes": 3707, "sha256": "8f701fd0fdbfc05436f36752f885dce6996d23adc4ecca76a446ebfb676fff5f"},
+ "judge:scene:runs/run/artifacts/renders/r00/input": {"bytes": 3688, "sha256": "14a0caff9e08f697edf414144628fdc7b3fe60838808ece0f990ef85d00e9cb2"},
"judge:scene:runs/run/artifacts/renders/r00/messages": {"bytes": 4930, "sha256": "562ef4bfc6b24c34604482cb73a02fb4b01da5c0e70685e3a1a4e1dda616a827"},
"judge:scene:runs/run/artifacts/renders/r00/sampling": {"bytes": 14069, "sha256": "9f5073025f0011d617d13bbde6605bf7358111b7700df92c41c23035482f789b"},
"judge:scene:runs/run/artifacts/renders/r00/system": {"bytes": 21572, "sha256": "eee5e34a58a91720755be68ed4d39cf127c634a484a4948e77c77ecce7e8cd16"},
@@ -736,19 +736,19 @@
"job:refine@r01/prompt": {"bytes": 7038, "sha256": "7567826ea7793e5f05f355c236b5984aa89abde8942f86b2a9e8a2e1881096c6"},
"job:refine@r01/scope": {"bytes": 426, "sha256": "3a39df2f37665e52ae78549431ea567fa9591605332e152cd7d41d2b58914893"},
"job:refine@r01/system": {"bytes": 2865, "sha256": "e8bdde0b2251c9f422c9a02c8c457543798280f5215e740312ffb73f1d0d867f"},
- "judge:static_object:runs/run/_cand/c0/artifacts/renders/r00/quick/input": {"bytes": 5480, "sha256": "d0f5d8618797710a5a8672d0e150fe844e47964c684f564870815881b8fccd22"},
+ "judge:static_object:runs/run/_cand/c0/artifacts/renders/r00/quick/input": {"bytes": 5423, "sha256": "8c8d53f300274af674ca6b0fad7eecd5b6785236b8bd7c6b805192aabbbbc65c"},
"judge:static_object:runs/run/_cand/c0/artifacts/renders/r00/quick/messages": {"bytes": 4246, "sha256": "029c3c03fe1f78f6f0d8b6b1b6bb4825d2851b55595a3c15c1a03f052b0b9771"},
"judge:static_object:runs/run/_cand/c0/artifacts/renders/r00/quick/sampling": {"bytes": 11371, "sha256": "275368055f7250512e426b330f9720fcfbdad1d1ea1a739102ab9f3521a3fb25"},
"judge:static_object:runs/run/_cand/c0/artifacts/renders/r00/quick/system": {"bytes": 15178, "sha256": "1becebca9df2996a380a8bedede59de191c58f5021c6f687a3dfd5abc673c4bc"},
- "judge:static_object:runs/run/_cand/c1/artifacts/renders/r00/quick/input": {"bytes": 5480, "sha256": "634a91a0f2eb65344528e8d83e744d8eba81d77d1b8697885a023f546199bef5"},
+ "judge:static_object:runs/run/_cand/c1/artifacts/renders/r00/quick/input": {"bytes": 5423, "sha256": "ac7c550b34c6dda878a8ae3189524ce95bef53cdd99e1d9c42055e5f5f57b028"},
"judge:static_object:runs/run/_cand/c1/artifacts/renders/r00/quick/messages": {"bytes": 4246, "sha256": "029c3c03fe1f78f6f0d8b6b1b6bb4825d2851b55595a3c15c1a03f052b0b9771"},
"judge:static_object:runs/run/_cand/c1/artifacts/renders/r00/quick/sampling": {"bytes": 11371, "sha256": "275368055f7250512e426b330f9720fcfbdad1d1ea1a739102ab9f3521a3fb25"},
"judge:static_object:runs/run/_cand/c1/artifacts/renders/r00/quick/system": {"bytes": 15178, "sha256": "1becebca9df2996a380a8bedede59de191c58f5021c6f687a3dfd5abc673c4bc"},
- "judge:static_object:runs/run/artifacts/renders/r00/input": {"bytes": 9043, "sha256": "dbaaabdbe8a66ecafe6caaf78c9d263dea54caa61f73b1c2a58356ac4eb232e8"},
+ "judge:static_object:runs/run/artifacts/renders/r00/input": {"bytes": 8986, "sha256": "263c78c5ef3b65de181e488b93344aac8ca350f211c0fdfcd6d25cd50c9fdec8"},
"judge:static_object:runs/run/artifacts/renders/r00/messages": {"bytes": 6699, "sha256": "7df542d34f969f456ab2fcd720b5ccf3a4547b1c57ed59a81df0a84a4d540b9a"},
"judge:static_object:runs/run/artifacts/renders/r00/sampling": {"bytes": 11371, "sha256": "275368055f7250512e426b330f9720fcfbdad1d1ea1a739102ab9f3521a3fb25"},
"judge:static_object:runs/run/artifacts/renders/r00/system": {"bytes": 15178, "sha256": "1becebca9df2996a380a8bedede59de191c58f5021c6f687a3dfd5abc673c4bc"},
- "judge:static_object:runs/run/artifacts/renders/r01/input": {"bytes": 10256, "sha256": "039d3ffebaf48a6e6adb10eeb65db4d8be1c1902af599781104b174ea82108f8"},
+ "judge:static_object:runs/run/artifacts/renders/r01/input": {"bytes": 10199, "sha256": "6512509cb2a85892bec8d1fdb69c60957061c9f95bd75f9da49d2d549002aad1"},
"judge:static_object:runs/run/artifacts/renders/r01/messages": {"bytes": 7124, "sha256": "a800f572433fa02006cc5a02c883359497949586963a0edeb799e047048d8f2c"},
"judge:static_object:runs/run/artifacts/renders/r01/sampling": {"bytes": 11371, "sha256": "275368055f7250512e426b330f9720fcfbdad1d1ea1a739102ab9f3521a3fb25"},
"judge:static_object:runs/run/artifacts/renders/r01/system": {"bytes": 15178, "sha256": "1becebca9df2996a380a8bedede59de191c58f5021c6f687a3dfd5abc673c4bc"},
@@ -786,11 +786,11 @@
"job:refine@r02/prompt": {"bytes": 7215, "sha256": "7d084dc936a0ca89af24a348321c9a88fd523c8353a034fe3bb37df511bcfe8e"},
"job:refine@r02/scope": {"bytes": 350, "sha256": "4e7e7adc88c67e32725d4fbf7d2f95b24d0166d57881f69cf543ab1c6936ea35"},
"job:refine@r02/system": {"bytes": 2205, "sha256": "2ca6cc8997115b5940dcef874355b65c82f25ee8000b964fe0ad96dce25ac0e8"},
- "judge:static_object:runs/run/artifacts/renders/r01/input": {"bytes": 9448, "sha256": "57f45cc8b04164a946635a087e7a5d925c8da6ba463e595f7f9723a3c8c49b4b"},
+ "judge:static_object:runs/run/artifacts/renders/r01/input": {"bytes": 9391, "sha256": "99babe8473b7d87bed3c2a33ef6609fe617960b927724e7d1b748736564f9eac"},
"judge:static_object:runs/run/artifacts/renders/r01/messages": {"bytes": 6808, "sha256": "afe220b9286ff63c1e78b62cf853ead93c525947a873c6eaea80d9094a9d483e"},
"judge:static_object:runs/run/artifacts/renders/r01/sampling": {"bytes": 11371, "sha256": "275368055f7250512e426b330f9720fcfbdad1d1ea1a739102ab9f3521a3fb25"},
"judge:static_object:runs/run/artifacts/renders/r01/system": {"bytes": 15178, "sha256": "1becebca9df2996a380a8bedede59de191c58f5021c6f687a3dfd5abc673c4bc"},
- "judge:static_object:runs/run/artifacts/renders/r02/input": {"bytes": 10657, "sha256": "c43068c990f633b1a5cfe0136b3c6255f37d49b4724aadc45bcafe17f5280b36"},
+ "judge:static_object:runs/run/artifacts/renders/r02/input": {"bytes": 10600, "sha256": "90d2ad98e2d79eba9e26bb5114c44ec23f00ab3f4e2202c1e5c6358ff9d30967"},
"judge:static_object:runs/run/artifacts/renders/r02/messages": {"bytes": 7233, "sha256": "d6a82300f22e09f4cc9f7d5a09ec89e5c6e9b300eab6b8899fc61648ac406c21"},
"judge:static_object:runs/run/artifacts/renders/r02/sampling": {"bytes": 11371, "sha256": "275368055f7250512e426b330f9720fcfbdad1d1ea1a739102ab9f3521a3fb25"},
"judge:static_object:runs/run/artifacts/renders/r02/system": {"bytes": 15178, "sha256": "1becebca9df2996a380a8bedede59de191c58f5021c6f687a3dfd5abc673c4bc"},
@@ -821,15 +821,15 @@
"job:refine@r02/prompt": {"bytes": 7174, "sha256": "3474b8ace6d7212c7430b362f161485f9243cdc6ef3118dc5d342c8720b85de5"},
"job:refine@r02/scope": {"bytes": 428, "sha256": "57e2f2633c63ce908a2842199385fe002439dcc7d6f68ffac0261d5be22cb8e1"},
"job:refine@r02/system": {"bytes": 3208, "sha256": "e24e99f321231e5587cae7f4fbbbcc50fbd254768f4f4c860c4dfa8f83804810"},
- "judge:static_object:runs/run/artifacts/renders/r00/input": {"bytes": 9242, "sha256": "24388c1c7e23be39af16276200ef38c874d6bd7a6f5efc2e8af4a639ac9ccaa9"},
+ "judge:static_object:runs/run/artifacts/renders/r00/input": {"bytes": 9185, "sha256": "69d608e7a5df7e8bde0e095fa488dd919724caff8a4c2b8b6f3d3470768d2cef"},
"judge:static_object:runs/run/artifacts/renders/r00/messages": {"bytes": 6756, "sha256": "447c252713a4d26c70c0b368c30217605f33543c3a329b6c32bee7099bdb1860"},
"judge:static_object:runs/run/artifacts/renders/r00/sampling": {"bytes": 11371, "sha256": "275368055f7250512e426b330f9720fcfbdad1d1ea1a739102ab9f3521a3fb25"},
"judge:static_object:runs/run/artifacts/renders/r00/system": {"bytes": 15178, "sha256": "1becebca9df2996a380a8bedede59de191c58f5021c6f687a3dfd5abc673c4bc"},
- "judge:static_object:runs/run/artifacts/renders/r01/input": {"bytes": 10455, "sha256": "3b59af59898d972cffb2178b608d9ffba14af4b76f5eb7b96993e3536cb8b29e"},
+ "judge:static_object:runs/run/artifacts/renders/r01/input": {"bytes": 10398, "sha256": "e65a4e44a4cf4d77bdb4e5d482cd06763e7b3de5780c64247dd991978fcef47c"},
"judge:static_object:runs/run/artifacts/renders/r01/messages": {"bytes": 7181, "sha256": "0ee41f64bf25c8490c05f3478f11afde83f2d8fc7f4753553d7497f1ef83543a"},
"judge:static_object:runs/run/artifacts/renders/r01/sampling": {"bytes": 11371, "sha256": "275368055f7250512e426b330f9720fcfbdad1d1ea1a739102ab9f3521a3fb25"},
"judge:static_object:runs/run/artifacts/renders/r01/system": {"bytes": 15178, "sha256": "1becebca9df2996a380a8bedede59de191c58f5021c6f687a3dfd5abc673c4bc"},
- "judge:static_object:runs/run/artifacts/renders/r02/input": {"bytes": 10451, "sha256": "e47aeb8f1cb8ad461a97940370c9fdd54d95a077c9a84df047a51167c66a5f64"},
+ "judge:static_object:runs/run/artifacts/renders/r02/input": {"bytes": 10394, "sha256": "c5fb5b97a21e23b5f24ee9e9926815b182f3522cd10d2237d58d01eea8d8b76f"},
"judge:static_object:runs/run/artifacts/renders/r02/messages": {"bytes": 7181, "sha256": "61b52dd59cc941e842e0e789faf595cd9ed24187eba10e7cc399e1ca792eb099"},
"judge:static_object:runs/run/artifacts/renders/r02/sampling": {"bytes": 11371, "sha256": "275368055f7250512e426b330f9720fcfbdad1d1ea1a739102ab9f3521a3fb25"},
"judge:static_object:runs/run/artifacts/renders/r02/system": {"bytes": 15178, "sha256": "1becebca9df2996a380a8bedede59de191c58f5021c6f687a3dfd5abc673c4bc"},
@@ -857,11 +857,11 @@
"job:refine@r01/prompt": {"bytes": 8861, "sha256": "40ce1992b720f3a98f57dedeab35b70590f3f9a49ef111a544a300896fa39a35"},
"job:refine@r01/scope": {"bytes": 644, "sha256": "f5e0c8fc1e3c0527734f358ad11929cedc505d872e61bbbfa4926d2cf5f4a1d0"},
"job:refine@r01/system": {"bytes": 3208, "sha256": "e24e99f321231e5587cae7f4fbbbcc50fbd254768f4f4c860c4dfa8f83804810"},
- "judge:static_object:runs/run/artifacts/renders/r00/input": {"bytes": 11147, "sha256": "cea087f4cc7335c78c7dbe31cd08285d097c7627cb53151853c3d882639b6f8b"},
+ "judge:static_object:runs/run/artifacts/renders/r00/input": {"bytes": 10977, "sha256": "a62a659e7ad6910734d575b2d919e05039e58646ed1759c25cf9f8609f67914e"},
"judge:static_object:runs/run/artifacts/renders/r00/messages": {"bytes": 7002, "sha256": "90d9f0c3e869c20608b23834b3a93df8242b1dd3324fdfda3b91de9542337eb0"},
"judge:static_object:runs/run/artifacts/renders/r00/sampling": {"bytes": 11137, "sha256": "3b6a70558e21e796673b65818721913f7720c6ee7e31dd35225cb4ca1bdcf20e"},
"judge:static_object:runs/run/artifacts/renders/r00/system": {"bytes": 15178, "sha256": "1becebca9df2996a380a8bedede59de191c58f5021c6f687a3dfd5abc673c4bc"},
- "judge:static_object:runs/run/artifacts/renders/r01/input": {"bytes": 12332, "sha256": "6879d4d19d95bb191dba70f85960b7fc5a2b385d850cdbc7b5f57a0f86a2e6e3"},
+ "judge:static_object:runs/run/artifacts/renders/r01/input": {"bytes": 12162, "sha256": "0bb3df066a50afdd9d13b3be5f69e3eaad404badf6954003005db702c095bffc"},
"judge:static_object:runs/run/artifacts/renders/r01/messages": {"bytes": 7429, "sha256": "70fca81684fcce477eadc3e0d5c329b3dcc31dcc527958abf29e484a0d555a7c"},
"judge:static_object:runs/run/artifacts/renders/r01/sampling": {"bytes": 11137, "sha256": "3b6a70558e21e796673b65818721913f7720c6ee7e31dd35225cb4ca1bdcf20e"},
"judge:static_object:runs/run/artifacts/renders/r01/system": {"bytes": 15178, "sha256": "1becebca9df2996a380a8bedede59de191c58f5021c6f687a3dfd5abc673c4bc"},
@@ -888,11 +888,11 @@
"job:refine@r01/prompt": {"bytes": 7012, "sha256": "9db3905eac671203349cd349d93748e1abadf89a9f584948cf3e64f0527abbcd"},
"job:refine@r01/scope": {"bytes": 428, "sha256": "186e31cec33c5144bbc9713eb76374ef7608534682433c9a9288191efc417ec4"},
"job:refine@r01/system": {"bytes": 3208, "sha256": "e24e99f321231e5587cae7f4fbbbcc50fbd254768f4f4c860c4dfa8f83804810"},
- "judge:static_object:runs/run/artifacts/renders/r00/input": {"bytes": 9043, "sha256": "e6f38103435ee5364a2bf61e2caedd75d4f4a7cf77c7d03d38d202bd8f73c1fa"},
+ "judge:static_object:runs/run/artifacts/renders/r00/input": {"bytes": 8986, "sha256": "2e7532a1396d719d97f1a606ef4b0a827d8a81ce2c43dd2dfe9a54784ebbbabf"},
"judge:static_object:runs/run/artifacts/renders/r00/messages": {"bytes": 6699, "sha256": "6e3964ce3478de892b3c5dde9eb3154e9a54c2a669315a8e6f4b808c15dd930c"},
"judge:static_object:runs/run/artifacts/renders/r00/sampling": {"bytes": 11371, "sha256": "275368055f7250512e426b330f9720fcfbdad1d1ea1a739102ab9f3521a3fb25"},
"judge:static_object:runs/run/artifacts/renders/r00/system": {"bytes": 15178, "sha256": "1becebca9df2996a380a8bedede59de191c58f5021c6f687a3dfd5abc673c4bc"},
- "judge:static_object:runs/run/artifacts/renders/r01/input": {"bytes": 10256, "sha256": "211aff02135e7034dd64d7c3c0a812acfad49abb4b87b6a441ddbb355d9e9117"},
+ "judge:static_object:runs/run/artifacts/renders/r01/input": {"bytes": 10199, "sha256": "c3778208bce04b21f24748b8549486a8e92378eaae50fb9eff12b48a01d3a156"},
"judge:static_object:runs/run/artifacts/renders/r01/messages": {"bytes": 7124, "sha256": "0e3a5d3ed25799eec213d2d7fd92ae5009e4d88d5a818eeaf0aef3de8d47b932"},
"judge:static_object:runs/run/artifacts/renders/r01/sampling": {"bytes": 11371, "sha256": "275368055f7250512e426b330f9720fcfbdad1d1ea1a739102ab9f3521a3fb25"},
"judge:static_object:runs/run/artifacts/renders/r01/system": {"bytes": 15178, "sha256": "1becebca9df2996a380a8bedede59de191c58f5021c6f687a3dfd5abc673c4bc"},
@@ -921,11 +921,11 @@
"chat:planner:planner/messages": {"bytes": 1354, "sha256": "5b1a589e27562a3668649aed4be4857a34804a2acdc140eb8fb3630d970f71a6"},
"chat:planner:planner/sampling": {"bytes": 6272, "sha256": "04dbfbbd42d8b93a12571a1908beacc7eb471e6f2c0fc077371aa86e2a99fb40"},
"chat:planner:planner/system": {"bytes": 18287, "sha256": "0e78778a5c81b420699c7148ee039722877c818cea9844f793d543fccc8ef97b"},
- "judge:static_object:runs/run/artifacts/renders/r00/input": {"bytes": 9060, "sha256": "9db5de7f375d60e4e56f9677affd8c4fbe4261f62d0111d1c021f08dcd5690bf"},
+ "judge:static_object:runs/run/artifacts/renders/r00/input": {"bytes": 9003, "sha256": "0b11a734774d2ee7cfec89359d4ad3c4bbb26ed6c6e5250da6db20478c0228c4"},
"judge:static_object:runs/run/artifacts/renders/r00/messages": {"bytes": 6699, "sha256": "6e3964ce3478de892b3c5dde9eb3154e9a54c2a669315a8e6f4b808c15dd930c"},
"judge:static_object:runs/run/artifacts/renders/r00/sampling": {"bytes": 11371, "sha256": "275368055f7250512e426b330f9720fcfbdad1d1ea1a739102ab9f3521a3fb25"},
"judge:static_object:runs/run/artifacts/renders/r00/system": {"bytes": 15178, "sha256": "1becebca9df2996a380a8bedede59de191c58f5021c6f687a3dfd5abc673c4bc"},
- "judge:static_object:runs/run/artifacts/renders/r01/input": {"bytes": 10273, "sha256": "97bda1184c1cd78145113857ec2e8917f6f51bd969f028f9dac064b1ee9dd0bb"},
+ "judge:static_object:runs/run/artifacts/renders/r01/input": {"bytes": 10216, "sha256": "bb40c3ba9b266e7301b8c5e6ade2a20b306acd2d535a0a002de19c46fa745dbf"},
"judge:static_object:runs/run/artifacts/renders/r01/messages": {"bytes": 7124, "sha256": "0e3a5d3ed25799eec213d2d7fd92ae5009e4d88d5a818eeaf0aef3de8d47b932"},
"judge:static_object:runs/run/artifacts/renders/r01/sampling": {"bytes": 11371, "sha256": "275368055f7250512e426b330f9720fcfbdad1d1ea1a739102ab9f3521a3fb25"},
"judge:static_object:runs/run/artifacts/renders/r01/system": {"bytes": 15178, "sha256": "1becebca9df2996a380a8bedede59de191c58f5021c6f687a3dfd5abc673c4bc"},
diff --git a/harness/tests/prompts/manifest.py b/harness/tests/prompts/manifest.py
index c325e873..e0f5804c 100644
--- a/harness/tests/prompts/manifest.py
+++ b/harness/tests/prompts/manifest.py
@@ -189,7 +189,7 @@ class ManifestServices(FakeServices):
def chat_model(self, model_id: str) -> Any:
return self.rig.model
- def judge(self, rubric: str, model_id: str, n_samples: int = 1) -> Any:
+ def judge(self, track: Any, rubric: str, model_id: str, n_samples: int = 1, *, references: bool = False) -> Any:
self.judges.append(StableJudge(rubric, self.rig.scores, self.rig.targets))
return self.judges[-1]