Official Codebase of 3DCodeVerse
Python
timing: every step of a run is a StepTime, and the run's minutes leave out provider errors
gaoypengb6baea839e8c
diff --git a/harness/codeverse3d/contracts/run.py b/harness/codeverse3d/contracts/run.pyindex c4d208aa..dd298044 100644--- a/harness/codeverse3d/contracts/run.py+++ b/harness/codeverse3d/contracts/run.py@@ -197,6 +197,25 @@ class RunStatus(StrEnum):return cls.STOPPED if value in ("passed", "plateau") else None+class StepTime(BaseModel):+ """One timed step of a run: the clock it took and the part of that clock it lost to+ provider errors — 503 / 429 / overloaded / timeout retries and their back-off, inside a+ vendor CLI (``AgentResult.provider_wait_s``) or an API call (``cost.instrument``). The+ run's minutes are the sum of ``wall_s - lost_s`` over its steps (owner, 2026-09-22).+ Parallel work — fan-out sessions, judge samples, the scene's assets beside its env — is+ ONE step, so no second of the clock is counted twice."""++ step: str = Field(description="plan | skeleton | textures | assets+env+layouts | zones | assemble | candidates"+ " | generate | build | gates | render | judge")+ round: int | None = Field(default=None, description="the round the step belongs to; None = a run-level stage")+ wall_s: float = 0.0+ lost_s: float = Field(default=0.0, description="seconds of wall_s lost to provider errors")++ @property+ def active_s(self) -> float:+ return max(0.0, self.wall_s - self.lost_s)++class RoundRecord(BaseModel):index: intkind: str = Field(description="baseline | refine | repair | texture | asset:<name> ...")@@ -212,13 +231,24 @@ class RoundRecord(BaseModel):skills: SkillsUsage | None = Field(default=None, description="skills attached to this round's sessions, and which were actually read")started_at: datetime = Field(default_factory=lambda: datetime.now(UTC))- duration_s: float = 0.0+ duration_s: float = Field(default=0.0, description="the round's clock, provider errors included")+ steps: list[StepTime] | None = Field(+ default=None, description="the round's timed steps (generate / build / gates / render / judge); "+ "None = recorded before 2026-09-22")notes: str = ""@propertydef score(self) -> float | None:return self.judgment.overall if self.judgment else None+ @property+ def minutes(self) -> float:+ """The round's minutes: its steps' ``wall_s - lost_s``. A round recorded before step+ timing (``steps is None``) counts its own clock, ``duration_s``."""+ if self.steps is None:+ return self.duration_s / 60.0+ return sum(s.active_s for s in self.steps) / 60.0+# --------------------------------------------------------------------------- telemetry (settings + cost)class RoleSettings(BaseModel):@@ -351,4 +381,24 @@ class RunRecord(BaseModel):error: str = ""telemetry: RunTelemetry | None = Field(default=None, description="settings snapshot + cost ledger + environment (written by finalize_record)")+ steps: list[StepTime] | None = Field(+ default=None, description="the run-level timed steps (plan, the prepare stages, best-of-N, the part of "+ "an aborted round that ran); None = recorded before 2026-09-22")extra: dict[str, Any] = Field(default_factory=dict)++ @property+ def minutes(self) -> float | None:+ """THE minutes of a run, the one number every reader reports: the ``wall_s - lost_s``+ of every step, run-level and round, so time lost to 503 / 429 / token-limit errors+ does not count (owner, 2026-09-22). It is a statistic, not the budget: the+ ``max_minutes`` clock (``orchestrator.BudgetGuard``) counts every active second,+ errors included, because it must stop a run even in a 503 storm.++ A record written before step timing (``steps is None``) counts its own clock+ instead: the budget's active minutes, ``extra["budget"]["elapsed_min"]`` (None when+ even that is missing)."""+ if self.steps is None:+ budget = self.extra.get("budget")+ elapsed = budget.get("elapsed_min") if isinstance(budget, dict) else None+ return float(elapsed) if isinstance(elapsed, (int, float)) else None+ return sum(s.active_s for s in self.steps) / 60.0 + sum(r.minutes for r in self.rounds)diff --git a/harness/codeverse3d/cost/instrument.py b/harness/codeverse3d/cost/instrument.pyindex 5e131c80..80fd2037 100644--- a/harness/codeverse3d/cost/instrument.py+++ b/harness/codeverse3d/cost/instrument.py@@ -51,6 +51,7 @@ from codeverse3d.cost.ledger import (open_run_ledger,record_call,)+from codeverse3d.cost.tally import Tally, book_time, open_talliesfrom codeverse3d.cost.types import Role, Stage, stage_for_labellog = logging.getLogger(__name__)@@ -85,15 +86,19 @@ class MeteredChatModel:def generate(self, request: ChatRequest) -> ChatResponse:call_id = uuid4().hex # joins the per-attempt rows to the call's logical row# captured NOW: a hedge loser lands after this call returned, in a thread with- # no context — neither the run's ledger, its name, nor the stage/role/round- # attribution (the loser's row used to fall back to the bare label) is reachable+ # no context — neither the run's ledger, its name, the stage/role/round+ # attribution (the loser's row used to fall back to the bare label) nor the+ # open tallies are reachableledger = default_ledger()run = run_binding().runctx = attribute(label=request.label)+ tallies = open_tallies()booked: list[int] = [] # round-trips that already billed themselves as "extra"+ trips = _RoundTrips()def attempt_row(rec: AttemptRecord) -> None:- if self._record_attempt(request, call_id, rec, ledger, run, ctx):+ trips.add(rec)+ if self._record_attempt(request, call_id, rec, ledger, run, ctx, tallies):booked.append(rec.attempt)t0 = time.perf_counter()@@ -104,19 +109,21 @@ class MeteredChatModel:# a failed call may still have been billed: ModelError.usage carries what the# provider charged (a bad-JSON reply costs like a good one) — but only when# the round-trips did not already book it, or the last one counts twice+ ms = int((time.perf_counter() - t0) * 1000)usage = getattr(exc, "usage", None)usage = usage if isinstance(usage, Usage) else Usage(backend=self.provider, model=self.model)+ book_time(ms / 1000, trips.lost_ms(ms, usage.latency_ms) / 1000)if booked:usage = Usage(backend=usage.backend, model=usage.model, latency_ms=usage.latency_ms)- self._record(usage, request, outcome=_outcome(exc),- ms=int((time.perf_counter() - t0) * 1000),+ self._record(usage, request, outcome=_outcome(exc), ms=ms,attempts=getattr(exc, "attempts", 0), call_id=call_id)raise+ ms = int((time.perf_counter() - t0) * 1000)+ book_time(ms / 1000, trips.lost_ms(ms, resp.usage.latency_ms) / 1000)# which key served it and how many round-trips it took (gemini.py puts both in# ``raw``); until 2026-08-26 no telemetry row carried either, so the per-key# distribution of calls could only be probed, never read- self._record(resp.usage, request, outcome="ok",- ms=int((time.perf_counter() - t0) * 1000),+ self._record(resp.usage, request, outcome="ok", ms=ms,key=resp.raw.get("key"), attempts=resp.raw.get("attempts", 0),call_id=call_id)return resp@@ -133,7 +140,7 @@ class MeteredChatModel:def _record_attempt(self, request: ChatRequest, call_id: str, rec: AttemptRecord,ledger: CostLedger | None, run: str = "",- ctx: CallContext | None = None) -> bool:+ ctx: CallContext | None = None, tallies: tuple[Tally, ...] | None = None) -> bool:"""One row per round-trip. A round-trip that was DISCARDED and still cost money(a billed-but-invalid reply, a hedge loser that landed) is money nothing elserecords, so it goes in as ``source="extra"`` and counts in every total; the rest@@ -150,13 +157,42 @@ class MeteredChatModel:latency_ms=rec.usage.latency_ms,source="extra" if extra else "attempt", key=_key_suffix(rec.key),call_id=call_id, attempt=rec.attempt, discarded=rec.discarded,- ledger=ledger)+ ledger=ledger, tallies=tallies)except Exception as e: # pragma: no cover - accounting must never break a calllog.debug("cost: could not record attempt %s#%d: %s", request.label, rec.attempt, e)return Falsereturn extra+class _RoundTrips:+ """The round-trips of ONE logical call, as the backend reported them through the attempt+ sink (``models.retry.rotate_with_retries``), reduced to what the minutes need: how long+ the ones that got an answer took, and whether any failed without one."""++ def __init__(self) -> None:+ self.answered_ms = 0 # the winner, and a reply that came back unusable (bad JSON, empty)+ self.failed = False # a round-trip with no answer at all: 503 / 429 / timeout / transport / dead key+ self.seen = False++ def add(self, rec: AttemptRecord) -> None:+ self.seen = True+ if rec.discarded and rec.outcome == "ok":+ return # a hedge loser that answered ran beside the winner, not before it+ if rec.usage.latency_ms:+ self.answered_ms += rec.usage.latency_ms+ elif rec.outcome != "ok":+ self.failed = True++ def lost_ms(self, wall_ms: int, answer_ms: int) -> int:+ """The call's milliseconds lost to provider errors: everything outside its answered+ round-trips once one failed without an answer — its failed tries, their back-off and+ the key waits between them. A backend that reports no round-trips (the SDK adapters)+ is charged everything outside the answer it returned."""+ if not self.seen:+ return max(0, wall_ms - answer_ms)+ return max(0, wall_ms - self.answered_ms) if self.failed else 0++def _key_suffix(key: object) -> str:"""The last 4 chars of an API key as ``…ab12`` — the ledger never holds more."""s = str(key or "").strip()@@ -203,8 +239,11 @@ class MeteredAgent:t0 = time.perf_counter()with call_context(round=job.round, stage=stage, role=Role.GENERATOR, label=job.label):result = self._inner.run(job)+ wall_s = time.perf_counter() - t0+ # the CLI's own retries of 503 / 429 / token-limit errors, read from its log+ book_time(wall_s, float(result.provider_wait_s or 0.0))# an opaque CLI: its session row is the only record- self._record_session(job, result, stage, int((time.perf_counter() - t0) * 1000))+ self._record_session(job, result, stage, int(wall_s * 1000))return resultdef _record_session(self, job: AgentJob, result: AgentResult, stage: Stage, ms: int) -> None:diff --git a/harness/codeverse3d/cost/ledger.py b/harness/codeverse3d/cost/ledger.pyindex 29e9fc72..d217ff61 100644--- a/harness/codeverse3d/cost/ledger.py+++ b/harness/codeverse3d/cost/ledger.py@@ -27,6 +27,7 @@ from typing import Anyfrom codeverse3d.contracts.common import Usagefrom codeverse3d.cost.context import CallContext, _stage, attribute+from codeverse3d.cost.tally import Tally, book_usagefrom codeverse3d.cost.types import (CallCost,CostBucket,@@ -256,9 +257,12 @@ def record_call(ledger: CostLedger | str | Path | None = None,ts: float | None = None,reprice: bool = False,+ tallies: tuple[Tally, ...] | None = None,**extra: Any,) -> CallCost:- """Append one priced call to the ledger and return the row.+ """Append one priced call to the ledger, book it into the open tallies+ (``cost.tally`` — or ``tallies``, captured by a caller whose row lands later in a+ thread with no context), and return the row.Callers pass the ``Usage`` they already hold::@@ -309,6 +313,11 @@ def record_call(target = ledger if isinstance(ledger, CostLedger) else (CostLedger(ledger) if ledger else default_ledger())if target is not None:target.append(row)+ if source != "attempt": # an attempt row's tokens are already on its call's own row+ book_usage(Usage(backend=row.backend, model=row.model, input_tokens=row.input_tokens,+ output_tokens=row.output_tokens, cached_tokens=row.cached_tokens,+ thoughts_tokens=row.thoughts_tokens, tool_calls=row.tool_calls,+ cost_usd=row.cost_usd, latency_ms=row.latency_ms), tallies)return rowdiff --git a/harness/codeverse3d/cost/tally.py b/harness/codeverse3d/cost/tally.pynew file mode 100644index 00000000..2951dd76--- /dev/null+++ b/harness/codeverse3d/cost/tally.py@@ -0,0 +1,105 @@+"""What the meters saw inside one block of work: its money and its lost time.++A :class:`Tally` is opened with :func:`tally` around a block (a round, a timed step) and+nests; everything metered inside the block books into every tally that is open:++* ``cost.ledger.record_call`` books each priced row, so ``Tally.usage`` is exactly the+ money ``telemetry/cost.jsonl`` holds for the block — the round's cost is read here, never+ accumulated a second way;+* the proxies in ``cost.instrument`` book each API call's and each agent session's wall+ clock and the part of it lost to provider errors (503 / 429 / overloaded / timeout+ retries and their back-off), per thread, so :meth:`Tally.lost_s` knows what the block+ lost even when its work ran side by side.++:func:`timed` turns one block into a ``StepTime`` row — the unit a run's minutes are+summed from (owner, 2026-09-22). The binding is a ``ContextVar``: ``proc.fan_out`` copies+the context, so a worker thread books into its caller's tallies.+"""++from __future__ import annotations++import threading+import time+from collections.abc import Iterator+from contextlib import contextmanager+from contextvars import ContextVar++from codeverse3d.contracts.common import Usage+from codeverse3d.contracts.run import StepTime+++class Tally:+ """The money and the lost seconds one block of work booked (see the module docstring)."""++ def __init__(self) -> None:+ self.usage = Usage()+ self._time: dict[int, list[float]] = {} # thread id → [wall_s, lost_s] of its metered calls+ self._lock = threading.Lock()++ def add_usage(self, usage: Usage) -> None:+ with self._lock:+ self.usage = self.usage + usage++ def add_time(self, wall_s: float, lost_s: float) -> None:+ with self._lock:+ t = self._time.setdefault(threading.get_ident(), [0.0, 0.0])+ t[0] += wall_s+ t[1] += min(lost_s, wall_s)++ def lost_s(self) -> float:+ """Seconds of the block's clock lost to provider errors. Calls in one thread ran one+ after the other, so their losses add up; threads ran side by side (fan-out sessions,+ judge samples, best-of-N candidates), so the block lost what its slowest thread would+ have been spared: ``max(wall) - max(wall - lost)`` over the threads."""+ with self._lock:+ if not self._time:+ return 0.0+ return max(w for w, _ in self._time.values()) - max(w - lost for w, lost in self._time.values())+++_open: ContextVar[tuple[Tally, ...]] = ContextVar("c3d_cost_tallies", default=())+++@contextmanager+def tally() -> Iterator[Tally]:+ """Open a tally for the block (nested ones all receive what the block books)."""+ t = Tally()+ token = _open.set(_open.get() + (t,))+ try:+ yield t+ finally:+ _open.reset(token)+++def open_tallies() -> tuple[Tally, ...]:+ """The tallies this execution context books into — captured by a caller whose work may+ land later in a thread with no context (a hedge loser's billed round-trip)."""+ return _open.get()+++def book_usage(usage: Usage, tallies: tuple[Tally, ...] | None = None) -> None:+ for t in open_tallies() if tallies is None else tallies:+ t.add_usage(usage)+++def book_time(wall_s: float, lost_s: float) -> None:+ for t in open_tallies():+ t.add_time(max(0.0, wall_s), max(0.0, lost_s))+++@contextmanager+def timed(step: str, into: list[StepTime], *, round_index: int | None = None) -> Iterator[Tally]:+ """Time ``step`` into ``into`` — its wall clock and what it lost — whether the block+ returns or raises. A step that took no time (a cached stage) leaves no row."""+ t0 = time.monotonic()+ with tally() as t:+ try:+ yield t+ finally:+ wall = time.monotonic() - t0+ if wall >= 0.01:+ into.append(StepTime(step=step, round=round_index, wall_s=round(wall, 2),+ lost_s=round(min(wall, t.lost_s()), 2)))+++__all__ = ["Tally", "book_time", "book_usage", "open_tallies", "tally", "timed"]diff --git a/harness/codeverse3d/orchestrator.py b/harness/codeverse3d/orchestrator.pyindex b65f0db9..eed26e4e 100644--- a/harness/codeverse3d/orchestrator.py+++ b/harness/codeverse3d/orchestrator.py@@ -25,9 +25,10 @@ from pydantic import BaseModel, Field, ValidationErrorfrom codeverse3d.contracts.artifacts import GateReport, Judgmentfrom codeverse3d.contracts.common import Budget, Usagefrom codeverse3d.contracts.plan import AcceptanceItem, Plan-from codeverse3d.contracts.run import RoundRecord, RunStatus+from codeverse3d.contracts.run import RoundRecord, RunStatus, StepTimefrom codeverse3d.conventions import to_snakefrom codeverse3d.cost.billing import bills_usd+from codeverse3d.cost.tally import timedfrom codeverse3d.proc import EventLogfrom codeverse3d.workspace import Workspace@@ -55,6 +56,8 @@ class RunState(BaseModel):status: RunStatus = RunStatus.PLANNINGstages: dict[str, StageState] = Field(default_factory=dict)+ steps: list[StepTime] = Field(default_factory=list, description="run-level timed steps, every session's "+ "(a stage, best-of-N, the part of an aborted round that ran) — the run's minutes")materialized_for: str = Field(default="", description="agent kind the workspace was materialised for")stop_reason: str = ""error: str = ""@@ -139,8 +142,11 @@ class StageRunner:*,inputs: Any = "",model: type[BaseModel] | None = None,+ timed_step: bool = True,) -> T:- """Run ``fn`` unless a cached result for the same ``inputs`` exists."""+ """Run ``fn`` unless a cached result for the same ``inputs`` exists. A stage that runs+ is a timed step of the run (``RunState.steps``) — unless ``timed_step=False``: a stage+ that runs beside its siblings is timed once, as their group, by its caller."""h = hash_inputs(inputs)path = self.result_path(name)prior = self.state.stages.get(name)@@ -166,7 +172,11 @@ class StageRunner:self.events.emit("stage.start", stage=name, inputs_hash=h)t0 = time.time()try:- result = fn()+ if timed_step:+ with timed(name, self.state.steps):+ result = fn()+ else:+ result = fn()except Exception as e:self.events.emit("stage.failed", stage=name, error=f"{type(e).__name__}: {e}",duration_s=round(time.time() - t0, 2))diff --git a/harness/codeverse3d/tracks/candidates.py b/harness/codeverse3d/tracks/candidates.pyindex b5775d1a..f72fe629 100644--- a/harness/codeverse3d/tracks/candidates.py+++ b/harness/codeverse3d/tracks/candidates.py@@ -29,6 +29,7 @@ from codeverse3d.contracts.artifacts import BuildResult, Measurement, RenderSetfrom codeverse3d.contracts.common import Usagefrom codeverse3d.contracts.run import RoundRecordfrom codeverse3d.conventions import OBJECT_VIEWS_QUICK+from codeverse3d.cost.tally import timedfrom codeverse3d.orchestrator import BudgetExceeded, gate_error_countfrom codeverse3d.proc import EventLog, fan_outfrom codeverse3d.tracks.common import RunContext@@ -72,16 +73,20 @@ def run_best_of_n(track: Any, ctx: RunContext, tasks: Sequence[GenerationTask],duration_s=cand.duration_s, cost_usd=round(rec.usage.cost_usd, 4), commit=rec.commit[:10])return cand- results = fan_out(list(range(n)), _one, max_workers=ctx.settings.limits.max_parallel_agents, label="candidates",- item_name=lambda k: f"c{k}")+ # the candidates run side by side: one step of the run's minutes (each candidate round+ # keeps its own steps in its sub-workspace)+ with timed("candidates", ctx.state.steps, round_index=0):+ results = fan_out(list(range(n)), _one, max_workers=ctx.settings.limits.max_parallel_agents,+ label="candidates", item_name=lambda k: f"c{k}")# one retry for candidates that crashed (transient 503s, parse failures) — not for budget stopsretry = [k for k, r in enumerate(results) if isinstance(r, Exception) and not isinstance(r, BudgetExceeded)]if retry and ctx.budget.ok():for k in retry:ctx.events.emit("candidate.retry", candidate=k, error=f"{type(results[k]).__name__}: {results[k]}"[:300])subs[k] = make_candidate_context(track, ctx, k)- again = fan_out(retry, _one, max_workers=ctx.settings.limits.max_parallel_agents, label="candidates-retry",- item_name=lambda k: f"c{k}")+ with timed("candidates", ctx.state.steps, round_index=0):+ again = fan_out(retry, _one, max_workers=ctx.settings.limits.max_parallel_agents, label="candidates-retry",+ item_name=lambda k: f"c{k}")for k, r in zip(retry, again, strict=True):results[k] = rrecords: list[CandidateRecord] = []diff --git a/harness/codeverse3d/tracks/lifecycle.py b/harness/codeverse3d/tracks/lifecycle.pyindex 6ce9a412..bf2dc4e0 100644--- a/harness/codeverse3d/tracks/lifecycle.py+++ b/harness/codeverse3d/tracks/lifecycle.py@@ -682,6 +682,7 @@ class BaseTrack:if a not in prior_aborted]return RunRecord(spec=ctx.spec, plan=ctx.plan, workspace=str(ctx.ws.root), status=status, rounds=rounds, total_usage=total,+ steps=list(ctx.state.steps),environment={"python": platform.python_version(), "host": platform.node(), "track": self.track.value,"language": ctx.language.value, "generator": ctx.agent_id},prompt_hashes={**prior_hashes, **dict(ctx.prompt_hashes)}, started_at=ctx.state.started_at,diff --git a/harness/codeverse3d/tracks/scene.py b/harness/codeverse3d/tracks/scene.pyindex edbb5169..d8cd93de 100644--- a/harness/codeverse3d/tracks/scene.py+++ b/harness/codeverse3d/tracks/scene.py@@ -39,6 +39,7 @@ from codeverse3d.contracts.common import TRACK_INFO, Trackfrom codeverse3d.contracts.plan import Plan, ScenePlan, ZonePlanfrom codeverse3d.contracts.run import RoundRecordfrom codeverse3d.conventions import to_snake+from codeverse3d.cost.tally import timedfrom codeverse3d.judges.base import judged_subsetfrom codeverse3d.languages.scene_threejs import zone_filefrom codeverse3d.orchestrator import StageRunner, TaskGroup, compact_instructions@@ -213,8 +214,9 @@ class SceneTrack(BaseTrack):ctx.extra["textures"] = runner.stage("textures", lambda: self._textures_stage(ctx), inputs={"plan": plan}) or {}stage_fns: dict[str, Any] = {"assets": lambda: run_asset_stage(ctx),"env": lambda: self._env_stage(ctx), "layouts": _layouts}- results = fan_out(list(stage_fns.items()), lambda kv: runner.stage(kv[0], kv[1], inputs=key),- max_workers=2, label="assets+env", item_name=lambda kv: kv[0])+ with timed("+".join(stage_fns), ctx.state.steps): # side by side: ONE step of the run's minutes+ results = fan_out(list(stage_fns.items()), lambda kv: runner.stage(kv[0], kv[1], inputs=key, timed_step=False),+ max_workers=2, label="assets+env", item_name=lambda kv: kv[0])staged = dict(zip(stage_fns, results, strict=True))first_exc = next((r for r in results if isinstance(r, Exception)), None)if first_exc is not None:diff --git a/harness/codeverse3d/tracks/steps.py b/harness/codeverse3d/tracks/steps.pyindex 2ae271c3..1dbaf6f1 100644--- a/harness/codeverse3d/tracks/steps.py+++ b/harness/codeverse3d/tracks/steps.py@@ -37,7 +37,8 @@ from codeverse3d.contracts.artifacts import ()from codeverse3d.contracts.common import Usagefrom codeverse3d.contracts.plan import AcceptanceItem, Plan-from codeverse3d.contracts.run import RoundRecord+from codeverse3d.contracts.run import RoundRecord, StepTime+from codeverse3d.cost.tally import timedfrom codeverse3d.judges.base import round_inputfrom codeverse3d.judges.rubrics import is_degradedfrom codeverse3d.orchestrator import BudgetExceeded, usage_delta@@ -115,13 +116,15 @@ def load_round_journal(ws: Workspace) -> list[RoundRecord]:return out-def run_generation_tasks(ctx: RunContext, tasks: Sequence[GenerationTask]) -> list[GenerationResult]:+def run_generation_tasks(ctx: RunContext, tasks: Sequence[GenerationTask], *,+ steps: list[StepTime] | None = None) -> list[GenerationResult]:"""Run tasks (parallel when > 1). Raises ``RoundFailed`` when none succeeded.``GenerationTask.phase`` sequences the round: tasks run in parallel WITHIN aphase, phases in ascending order. Every task is phase 0 unless a track saysotherwise, so this is a no-op for every existing caller; per-part scoped- generation uses it so the assembly session sees the part files first.+ generation uses it so the assembly session sees the part files first. Each+ phase is one ``generate`` step in ``steps``."""if not tasks:return []@@ -132,7 +135,7 @@ def run_generation_tasks(ctx: RunContext, tasks: Sequence[GenerationTask]) -> lifor ph in phases:picked = [(i, t) for i, t in enumerate(tasks) if t.phase == ph]try:- got = _run_phase(ctx, [t for _, t in picked])+ got = _run_phase(ctx, [t for _, t in picked], steps)except RoundFailed as e:# a phase in which nothing succeeded is not automatically a dead round:# the earlier phases may have written the parts. Record the failures@@ -146,19 +149,22 @@ def run_generation_tasks(ctx: RunContext, tasks: Sequence[GenerationTask]) -> liif not ok_any:raise RoundFailed.of(results_seq, "no generation task succeeded")return results_seq- return _run_phase(ctx, list(tasks))+ return _run_phase(ctx, list(tasks), steps)-def _run_phase(ctx: RunContext, tasks: Sequence[GenerationTask]) -> list[GenerationResult]:- """One parallel batch of generation tasks (the pre-phase behaviour, unchanged)."""+def _run_phase(ctx: RunContext, tasks: Sequence[GenerationTask],+ steps: list[StepTime] | None = None) -> list[GenerationResult]:+ """One parallel batch of generation tasks — one ``generate`` step: its sessions run side+ by side, so they share one clock."""def _one(task: GenerationTask) -> GenerationResult:return generate_for(ctx, task)# fan_out also for ONE task: a crashing generator (503 storm, parse error) becomes a failed# result → RoundFailed (a refine round then stops the run as no_change) instead of killing it.- results = fan_out(list(tasks), _one, max_workers=ctx.settings.limits.max_parallel_agents,- label="generate", item_name=lambda t: t.label)+ with timed("generate", [] if steps is None else steps, round_index=tasks[0].round if tasks else None):+ results = fan_out(list(tasks), _one, max_workers=ctx.settings.limits.max_parallel_agents,+ label="generate", item_name=lambda t: t.label)out: list[GenerationResult] = []budget_stop: Exception | None = Nonefor task, r in zip(tasks, results, strict=True):@@ -201,15 +207,16 @@ def run_round(clay views): the two knobs a best-of-N candidate turns (``candidates.run_best_of_n``calls it directly for each candidate in its sub-workspace)."""mark = ctx.budget.mark()+ steps: list[StepTime] = []try:return _run_round(ctx, index=index, kind=kind, tasks=tasks, pipeline=pipeline, instructions=instructions,previous=previous, files_hint=files_hint, extra_usage=extra_usage,- extra_notes=extra_notes)+ extra_notes=extra_notes, steps=steps)except BaseException as e:# the round died half-way (budget stop, 503 storm, RoundFailed). Whatever it# burned is already in the guard: report it so the round is not invisible.burned = usage_delta(ctx.budget.spent, mark)- record_aborted_round(ctx, index=index, kind=kind, usage=burned, error=f"{type(e).__name__}: {e}")+ record_aborted_round(ctx, index=index, kind=kind, usage=burned, error=f"{type(e).__name__}: {e}", steps=steps)raise@@ -227,8 +234,10 @@ def _run_round(extra_notes: Sequence[str] = (),render: RenderFn | None = None,geometry_views: bool = True,+ steps: list[StepTime] | None = None,) -> RoundRecord:t0 = time.time()+ steps = [] if steps is None else stepsctx.events.emit("round.start", round=index, kind=kind, n_tasks=len(tasks))rec = RoundRecord(index=index, kind=kind, agent_backend=ctx.agent_id, instructions=list(instructions),started_at=datetime.now(UTC))@@ -243,7 +252,7 @@ def _run_round(# generated with a strictly smaller library than a --candidates 1 baseline (review 2026-08-29)skill_kind = "baseline" if kind == "candidate" else kindskills_hook.attach_for_round(ctx, index=index, kind=skill_kind)- gens = run_generation_tasks(ctx, skills_hook.with_inlined_skill(ctx, tasks))+ gens = run_generation_tasks(ctx, skills_hook.with_inlined_skill(ctx, tasks), steps=steps)turns = 0for g in gens:usage = usage + g.usage@@ -254,8 +263,9 @@ def _run_round(cost["generate"] = round(sum(g.usage.cost_usd for g in gens), 6)ctx.ws.commit(f"r{index:02d} {kind}: generated")- outcome: RepairOutcome = build_with_repair(ctx, round_index=index, label=f"r{index:02d}_{kind}",- files_hint=list(files_hint))+ with timed("build", steps, round_index=index):+ outcome: RepairOutcome = build_with_repair(ctx, round_index=index, label=f"r{index:02d}_{kind}",+ files_hint=list(files_hint))usage = usage + outcome.usageif outcome.usage.cost_usd:cost["repair"] = round(outcome.usage.cost_usd, 6)@@ -268,10 +278,12 @@ def _run_round()if outcome.build.ok:- rec.measurement = pipeline.measure(ctx, outcome.build)- gates.extend(pipeline.gates(ctx, index, outcome.build, rec.measurement))+ with timed("gates", steps, round_index=index):+ rec.measurement = pipeline.measure(ctx, outcome.build)+ gates.extend(pipeline.gates(ctx, index, outcome.build, rec.measurement))try:- rec.renders = (render or pipeline.render)(ctx, index, outcome.build, rec.measurement)+ with timed("render", steps, round_index=index):+ rec.renders = (render or pipeline.render)(ctx, index, outcome.build, rec.measurement)except RenderError as e:# A render that times out (measured 2026-08-26, art_verify camera_tripod: the# refine round built in 1.2 s, then render_glb.mjs hit its 330 s timeout with@@ -284,7 +296,8 @@ def _run_round(ctx.events.emit("render.failed", round=index, error=str(e)[:400])post = getattr(pipeline, "post_render_gates", None)if callable(post) and rec.renders is not None:- gates.extend(post(ctx, index, rec.renders))+ with timed("gates", steps, round_index=index):+ gates.extend(post(ctx, index, rec.renders))# the build's own reports (scene_probe + shader_preflight, gl_frames) after the# track's: the judge's warning list is capped and reads them in this ordergates.extend(outcome.build.gates)@@ -296,8 +309,9 @@ def _run_round(notes.append(f"judge skipped ({skip})")ctx.events.emit("judge.skipped", round=index, reason=skip)else:- rec.judgment, judge_usage = _judge(ctx, pipeline, index, outcome.build, gates, rec, previous, notes,- geometry_views=geometry_views)+ with timed("judge", steps, round_index=index):+ rec.judgment, judge_usage = _judge(ctx, pipeline, index, outcome.build, gates, rec, previous, notes,+ geometry_views=geometry_views)if rec.judgment is not None or judge_usage.cost_usd:# a degraded/crashed verdict was still PAID: fold its usage into the# round's usage and cost["judge"] so rec.usage and the cost.round event@@ -323,6 +337,7 @@ def _run_round(keep_round_artifacts(ctx.ws, index)rec.usage = usagerec.duration_s = round(time.time() - t0, 2)+ rec.steps = stepsrec.notes = "; ".join(notes)ctx.ws.write_json(round_record_path(ctx, index), rec)ctx.events.emit("round.done", round=index, kind=kind, commit=rec.commit[:10], score=rec.score,@@ -400,16 +415,19 @@ def _waste_flag(*, build_ok: bool, judged: bool, aborted: str) -> tuple[bool, streturn False, ""-def record_aborted_round(ctx: RunContext, *, index: int, kind: str, usage: Usage, error: str) -> None:+def record_aborted_round(ctx: RunContext, *, index: int, kind: str, usage: Usage, error: str,+ steps: Sequence[StepTime] = ()) -> None:"""A round that raised half-way still reports what it burned.Its record is written next to the round records as ``aborted_rNN.json`` (never``rNN.json``: the round did not happen, and ``load_round_journal`` must not- resume from it), a ``cost.round`` event is emitted, and the usage is remembered- in ``ctx.extra`` so the run record can carry it. Accounting a stop must never- raise on top of the stop that is already happening."""+ resume from it), a ``cost.round`` event is emitted, the usage is remembered+ in ``ctx.extra`` so the run record can carry it, and the steps it did run join the+ run-level steps (``RunState.steps``): their minutes were spent. Accounting a stop+ must never raise on top of the stop that is already happening."""try:- rec = RoundRecord(index=index, kind=kind, agent_backend=ctx.agent_id, usage=usage,+ ctx.state.steps.extend(steps)+ rec = RoundRecord(index=index, kind=kind, agent_backend=ctx.agent_id, usage=usage, steps=list(steps),notes=f"aborted: {error}")path = ctx.ws.root / "rounds" / f"aborted_r{index:02d}.json"path.parent.mkdir(parents=True, exist_ok=True)@@ -491,7 +509,9 @@ def rejudge_round(ctx: RunContext, pipeline: RoundPipeline, rec: RoundRecord, prreturn Falsectx.events.emit("judge.retry", round=rec.index)notes: list[str] = [rec.notes] if rec.notes else []- judgment, judge_usage = _judge(ctx, pipeline, rec.index, rec.build, list(rec.gates), rec, previous, notes)+ # a round recorded before step timing keeps counting its own clock (RoundRecord.minutes)+ with timed("judge", rec.steps if rec.steps is not None else [], round_index=rec.index):+ judgment, judge_usage = _judge(ctx, pipeline, rec.index, rec.build, list(rec.gates), rec, previous, notes)if judgment is None:# the retry failed AGAIN — but it was still paid for: keep the money and the# failure note on the persisted record so totals and resume can see them.diff --git a/harness/docs/ARCHITECTURE.md b/harness/docs/ARCHITECTURE.mdindex 4b53019c..ec948a7e 100644--- a/harness/docs/ARCHITECTURE.md+++ b/harness/docs/ARCHITECTURE.md@@ -163,6 +163,7 @@ codeverse3d/profiles.py (economy|balanced|quality; cli._common.resolve_dial is THE resolver)billing.py (SUBSCRIPTION_BACKENDS/bills_usd — which backends take real dollars,so the ledger bills real money and not list price; docs/COST.md §25)+ tally.py (a block's money = its ledger rows, its lost seconds → StepTime; COST §31)guard.py reconstruct.py (old runs)judges/ base.py (JudgeInput; `round_input` + `plan_summary(plan, language)`, the ONE payloadbuilder and plan digest the in-run judge, `3dcode judge`, calibration, the texturediff --git a/harness/tests/cost/test_tally.py b/harness/tests/cost/test_tally.pynew file mode 100644index 00000000..8847d6ad--- /dev/null+++ b/harness/tests/cost/test_tally.py@@ -0,0 +1,140 @@+"""Tallies: a block's money is its ledger rows, its minutes are its clock minus provider errors."""++from __future__ import annotations++import threading+import time+from pathlib import Path++import pytest++from codeverse3d.contracts.agent import AgentJob, AgentResult+from codeverse3d.contracts.chat import ChatMessage, ChatRequest, ChatResponse+from codeverse3d.contracts.common import Usage+from codeverse3d.contracts.run import StepTime+from codeverse3d.cost.context import AttemptRecord, attempt_sink+from codeverse3d.cost.instrument import MeteredAgent, MeteredChatModel, run_ledger+from codeverse3d.cost.ledger import load_ledger, record_call+from codeverse3d.cost.tally import Tally, tally, timed++REQ = ChatRequest(messages=[ChatMessage.user("x")], label="judge:static_object_v1:r00:s0")+++def test_a_tally_holds_exactly_the_ledger_rows_booked_inside_it(tmp_path: Path):+ u = Usage(backend="gemini", model="gemini-3.7-flash", input_tokens=1_000, output_tokens=10)+ with run_ledger(tmp_path, run="r"), tally() as outer:+ record_call(u, stage="plan")+ with tally() as inner:+ record_call(u, stage="judge")+ record_call(u, stage="judge", source="attempt") # forensics: its call's row carries it+ assert inner.usage.input_tokens == 1_000 and outer.usage.input_tokens == 2_000+ assert outer.usage.cost_usd == pytest.approx(sum(r.cost_usd for r in load_ledger(tmp_path)))+++def _side_by_side(*calls: tuple[float, float]) -> Tally:+ """Book one (wall, lost) per thread, every thread alive at once (idents are reused after exit)."""+ t, gate = Tally(), threading.Barrier(len(calls))++ def worker(wall: float, lost: float) -> None:+ gate.wait()+ t.add_time(wall, lost)+ gate.wait()++ threads = [threading.Thread(target=worker, args=c) for c in calls]+ for th in threads:+ th.start()+ for th in threads:+ th.join()+ return t+++def test_lost_time_adds_up_in_a_thread_and_takes_the_slowest_thread_side_by_side():+ t = Tally()+ t.add_time(60.0, 10.0)+ t.add_time(30.0, 5.0) # one thread, one call after the other+ assert t.lost_s() == pytest.approx(15.0)+ # A lost 100 of its 600 s; B took 550 without an error: the step would have ended at 550+ assert _side_by_side((600.0, 100.0), (550.0, 0.0)).lost_s() == pytest.approx(50.0)+ # B took 600 anyway: A's errors cost the step nothing+ assert _side_by_side((600.0, 100.0), (600.0, 0.0)).lost_s() == pytest.approx(0.0)+ # both sat out the same storm: the step lost it once, not twice+ assert _side_by_side((600.0, 100.0), (560.0, 100.0)).lost_s() == pytest.approx(100.0)+++def test_a_timed_step_is_one_row_with_its_wall_and_what_it_lost():+ steps: list[StepTime] = []+ with timed("judge", steps, round_index=2) as t:+ t.add_time(5.0, 2.0)+ time.sleep(0.03)+ (row,) = steps+ assert row.step == "judge" and row.round == 2 and row.wall_s >= 0.03+ assert row.lost_s == pytest.approx(row.wall_s), "lost is clipped to the step's own clock"+ with timed("skeleton", steps):+ pass+ assert len(steps) == 1, "a step that took no time leaves no row"+ with pytest.raises(RuntimeError), timed("build", steps):+ time.sleep(0.02)+ raise RuntimeError("the time was still spent")+ assert steps[-1].step == "build"+++class SinkChat:+ """A backend that reports its round-trips like ``GeminiModel`` does."""++ provider, model, id = "gemini", "gemini-3.7-flash", "gemini:gemini-3.7-flash"++ def __init__(self, trips: list[tuple[str, int, float]]):+ self.trips = trips # (outcome, answered latency ms, seconds the round-trip holds)++ def generate(self, request: ChatRequest) -> ChatResponse:+ sink = attempt_sink()+ last = len(self.trips)+ for i, (outcome, latency, hold) in enumerate(self.trips, 1):+ time.sleep(hold)+ sink(AttemptRecord(attempt=i, key="k", outcome=outcome, discarded=i != last,+ usage=Usage(model=self.model, latency_ms=latency)))+ return ChatResponse(text="ok", usage=Usage(model=self.model, input_tokens=10, latency_ms=self.trips[-1][1]))+++def _lost(chat: object, tmp_path: Path) -> tuple[float, float]:+ with run_ledger(tmp_path, run="r"), tally() as t:+ MeteredChatModel(chat).generate(REQ)+ (wall, lost), = t._time.values()+ return wall, lost+++def test_a_clean_call_loses_nothing_and_a_503_before_the_answer_is_lost(tmp_path: Path):+ wall, lost = _lost(SinkChat([("ok", 50, 0.05)]), tmp_path)+ assert lost == 0.0 and wall >= 0.05+ wall, lost = _lost(SinkChat([("5xx", 0, 0.10), ("ok", 40, 0.04)]), tmp_path)+ assert lost == pytest.approx(wall - 0.040, abs=0.02) and lost >= 0.09+ # a reply that came back unusable is the model's own time, not the provider's+ wall, lost = _lost(SinkChat([("skip", 60, 0.06), ("ok", 40, 0.04)]), tmp_path)+ assert lost == 0.0+++def test_a_backend_that_reports_no_round_trips_is_charged_everything_outside_its_answer(tmp_path: Path):+ class Sdk:+ provider, model, id = "openai", "gpt-5", "openai:gpt-5"++ def generate(self, request: ChatRequest) -> ChatResponse:+ time.sleep(0.08) # a retried attempt and its back-off, invisible from here+ return ChatResponse(text="ok", usage=Usage(model="gpt-5", latency_ms=20))++ wall, lost = _lost(Sdk(), tmp_path)+ assert lost == pytest.approx(wall - 0.020, abs=0.01)+++def test_an_agent_session_books_what_its_cli_lost_to_the_provider(tmp_path: Path):+ class Cli:+ kind, model, id = "gemini-cli", "gemini-3.7-flash", "gemini-cli:gemini-3.7-flash"++ def run(self, job: AgentJob) -> AgentResult:+ time.sleep(0.02)+ return AgentResult(ok=True, provider_wait_s=0.015, usage=Usage(backend="gemini-cli", input_tokens=5))++ with run_ledger(tmp_path, run="r"), tally() as t:+ MeteredAgent(Cli()).run(AgentJob(workspace=str(tmp_path), prompt="p", label="baseline", round=0))+ (wall, lost), = t._time.values()+ assert wall >= 0.02 and lost == pytest.approx(0.015)+ assert t.usage.input_tokens == 5, "the session row is booked into the tally like any ledger row"diff --git a/harness/tests/orchestrator_tracks/test_step_timing.py b/harness/tests/orchestrator_tracks/test_step_timing.pynew file mode 100644index 00000000..19a07d84--- /dev/null+++ b/harness/tests/orchestrator_tracks/test_step_timing.py@@ -0,0 +1,61 @@+"""A run's minutes are the sum of its steps' clocks minus what provider errors cost them+(owner, 2026-09-22) — THE number every reader reports."""++from __future__ import annotations++import time++from codeverse3d.contracts.agent import AgentJob, AgentResult+from codeverse3d.contracts.common import Language+from codeverse3d.contracts.run import RoundRecord, RunRecord, StepTime+from codeverse3d.cost.instrument import MeteredAgent+from codeverse3d.orchestrator import RunState+from codeverse3d.tracks.static_object import StaticObjectTrack+from codeverse3d.workspace import Workspace+from tests.orchestrator_tracks.conftest import make_spec+from tests.orchestrator_tracks.fakes import (+ FakeAgent,+ FakeJudge,+ FakeRuntime,+ FakeServices,+ _planner,+)+++class StormyAgent(FakeAgent):+ """A vendor CLI that spent part of every session retrying 503s."""++ def run(self, job: AgentJob) -> AgentResult:+ res = super().run(job)+ time.sleep(0.12)+ return res.model_copy(update={"provider_wait_s": 0.08})+++def test_every_step_is_timed_and_the_minutes_leave_out_the_provider_errors(tmp_path, chair_plan, settings):+ agent = MeteredAgent(StormyAgent(lambda job, ws: {"src/model.py": f"import bpy # r{job.round}\n"}))+ ws = Workspace(tmp_path / "runs" / "timed")+ track = StaticObjectTrack(services=FakeServices(), judge=FakeJudge(scores=(0.5, 0.6)), agent=agent,+ planner_model=_planner(chair_plan.model_dump(mode="json")), settings=settings,+ runtime=FakeRuntime(Language.BLENDER))+ rec = track.run(make_spec(language=Language.BLENDER, max_rounds=1), ws)+ for r in rec.rounds:+ (gen,) = [s for s in r.steps if s.step == "generate"]+ assert gen.round == r.index and gen.wall_s >= 0.12 and abs(gen.lost_s - 0.08) < 0.005+ assert rec.steps is not None and all(s.round is None for s in rec.steps)+ total = sum(s.active_s for s in rec.steps) + sum(s.active_s for r in rec.rounds for s in r.steps)+ assert abs(rec.minutes - total / 60) < 1e-9+ assert rec.minutes < sum(r.duration_s for r in rec.rounds) / 60, "the 503 retries are not in the minutes"+ # the step log survives on disk: the journal carries each round's, run_state the run's+ assert RoundRecord.model_validate(ws.read_json(ws.root / "rounds" / "r00.json")).steps == rec.rounds[0].steps+ assert RunState.load(ws).steps == rec.steps+++def test_a_record_from_before_step_timing_counts_its_own_clock():+ old = RunRecord.model_validate({"spec": make_spec().model_dump(mode="json"), "workspace": "w",+ "rounds": [{"index": 0, "kind": "baseline", "duration_s": 120.0}],+ "extra": {"budget": {"elapsed_min": 7.5}}})+ assert old.steps is None and old.minutes == 7.5 and old.rounds[0].minutes == 2.0+ assert RunRecord.model_validate({"spec": make_spec().model_dump(mode="json"), "workspace": "w"}).minutes is None+ # a run resumed across the change: new steps, and an old round that still counts its clock+ mixed = old.model_copy(update={"steps": [StepTime(step="plan", wall_s=90.0, lost_s=30.0)]})+ assert mixed.minutes == 1.0 + 2.0