Skip to content

Official Codebase of 3DCodeVerse

Python

timing: every step of a run is a StepTime, and the run's minutes leave out provider errors

gaoypengb6baea839e8c
diff --git a/harness/codeverse3d/contracts/run.py b/harness/codeverse3d/contracts/run.py
index c4d208aa..dd298044 100644
--- a/harness/codeverse3d/contracts/run.py
+++ b/harness/codeverse3d/contracts/run.py
@@ -197,6 +197,25 @@ class RunStatus(StrEnum):
return cls.STOPPED if value in ("passed", "plateau") else None
+class StepTime(BaseModel):
+ """One timed step of a run: the clock it took and the part of that clock it lost to
+ provider errors — 503 / 429 / overloaded / timeout retries and their back-off, inside a
+ vendor CLI (``AgentResult.provider_wait_s``) or an API call (``cost.instrument``). The
+ run's minutes are the sum of ``wall_s - lost_s`` over its steps (owner, 2026-09-22).
+ Parallel work — fan-out sessions, judge samples, the scene's assets beside its env — is
+ ONE step, so no second of the clock is counted twice."""
+
+ step: str = Field(description="plan | skeleton | textures | assets+env+layouts | zones | assemble | candidates"
+ " | generate | build | gates | render | judge")
+ round: int | None = Field(default=None, description="the round the step belongs to; None = a run-level stage")
+ wall_s: float = 0.0
+ lost_s: float = Field(default=0.0, description="seconds of wall_s lost to provider errors")
+
+ @property
+ def active_s(self) -> float:
+ return max(0.0, self.wall_s - self.lost_s)
+
+
class RoundRecord(BaseModel):
index: int
kind: str = Field(description="baseline | refine | repair | texture | asset:<name> ...")
@@ -212,13 +231,24 @@ class RoundRecord(BaseModel):
skills: SkillsUsage | None = Field(
default=None, description="skills attached to this round's sessions, and which were actually read")
started_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
- duration_s: float = 0.0
+ duration_s: float = Field(default=0.0, description="the round's clock, provider errors included")
+ steps: list[StepTime] | None = Field(
+ default=None, description="the round's timed steps (generate / build / gates / render / judge); "
+ "None = recorded before 2026-09-22")
notes: str = ""
@property
def score(self) -> float | None:
return self.judgment.overall if self.judgment else None
+ @property
+ def minutes(self) -> float:
+ """The round's minutes: its steps' ``wall_s - lost_s``. A round recorded before step
+ timing (``steps is None``) counts its own clock, ``duration_s``."""
+ if self.steps is None:
+ return self.duration_s / 60.0
+ return sum(s.active_s for s in self.steps) / 60.0
+
# --------------------------------------------------------------------------- telemetry (settings + cost)
class RoleSettings(BaseModel):
@@ -351,4 +381,24 @@ class RunRecord(BaseModel):
error: str = ""
telemetry: RunTelemetry | None = Field(
default=None, description="settings snapshot + cost ledger + environment (written by finalize_record)")
+ steps: list[StepTime] | None = Field(
+ default=None, description="the run-level timed steps (plan, the prepare stages, best-of-N, the part of "
+ "an aborted round that ran); None = recorded before 2026-09-22")
extra: dict[str, Any] = Field(default_factory=dict)
+
+ @property
+ def minutes(self) -> float | None:
+ """THE minutes of a run, the one number every reader reports: the ``wall_s - lost_s``
+ of every step, run-level and round, so time lost to 503 / 429 / token-limit errors
+ does not count (owner, 2026-09-22). It is a statistic, not the budget: the
+ ``max_minutes`` clock (``orchestrator.BudgetGuard``) counts every active second,
+ errors included, because it must stop a run even in a 503 storm.
+
+ A record written before step timing (``steps is None``) counts its own clock
+ instead: the budget's active minutes, ``extra["budget"]["elapsed_min"]`` (None when
+ even that is missing)."""
+ if self.steps is None:
+ budget = self.extra.get("budget")
+ elapsed = budget.get("elapsed_min") if isinstance(budget, dict) else None
+ return float(elapsed) if isinstance(elapsed, (int, float)) else None
+ return sum(s.active_s for s in self.steps) / 60.0 + sum(r.minutes for r in self.rounds)
diff --git a/harness/codeverse3d/cost/instrument.py b/harness/codeverse3d/cost/instrument.py
index 5e131c80..80fd2037 100644
--- a/harness/codeverse3d/cost/instrument.py
+++ b/harness/codeverse3d/cost/instrument.py
@@ -51,6 +51,7 @@ from codeverse3d.cost.ledger import (
open_run_ledger,
record_call,
)
+from codeverse3d.cost.tally import Tally, book_time, open_tallies
from codeverse3d.cost.types import Role, Stage, stage_for_label
log = logging.getLogger(__name__)
@@ -85,15 +86,19 @@ class MeteredChatModel:
def generate(self, request: ChatRequest) -> ChatResponse:
call_id = uuid4().hex # joins the per-attempt rows to the call's logical row
# captured NOW: a hedge loser lands after this call returned, in a thread with
- # no context — neither the run's ledger, its name, nor the stage/role/round
- # attribution (the loser's row used to fall back to the bare label) is reachable
+ # no context — neither the run's ledger, its name, the stage/role/round
+ # attribution (the loser's row used to fall back to the bare label) nor the
+ # open tallies are reachable
ledger = default_ledger()
run = run_binding().run
ctx = attribute(label=request.label)
+ tallies = open_tallies()
booked: list[int] = [] # round-trips that already billed themselves as "extra"
+ trips = _RoundTrips()
def attempt_row(rec: AttemptRecord) -> None:
- if self._record_attempt(request, call_id, rec, ledger, run, ctx):
+ trips.add(rec)
+ if self._record_attempt(request, call_id, rec, ledger, run, ctx, tallies):
booked.append(rec.attempt)
t0 = time.perf_counter()
@@ -104,19 +109,21 @@ class MeteredChatModel:
# a failed call may still have been billed: ModelError.usage carries what the
# provider charged (a bad-JSON reply costs like a good one) — but only when
# the round-trips did not already book it, or the last one counts twice
+ ms = int((time.perf_counter() - t0) * 1000)
usage = getattr(exc, "usage", None)
usage = usage if isinstance(usage, Usage) else Usage(backend=self.provider, model=self.model)
+ book_time(ms / 1000, trips.lost_ms(ms, usage.latency_ms) / 1000)
if booked:
usage = Usage(backend=usage.backend, model=usage.model, latency_ms=usage.latency_ms)
- self._record(usage, request, outcome=_outcome(exc),
- ms=int((time.perf_counter() - t0) * 1000),
+ self._record(usage, request, outcome=_outcome(exc), ms=ms,
attempts=getattr(exc, "attempts", 0), call_id=call_id)
raise
+ ms = int((time.perf_counter() - t0) * 1000)
+ book_time(ms / 1000, trips.lost_ms(ms, resp.usage.latency_ms) / 1000)
# which key served it and how many round-trips it took (gemini.py puts both in
# ``raw``); until 2026-08-26 no telemetry row carried either, so the per-key
# distribution of calls could only be probed, never read
- self._record(resp.usage, request, outcome="ok",
- ms=int((time.perf_counter() - t0) * 1000),
+ self._record(resp.usage, request, outcome="ok", ms=ms,
key=resp.raw.get("key"), attempts=resp.raw.get("attempts", 0),
call_id=call_id)
return resp
@@ -133,7 +140,7 @@ class MeteredChatModel:
def _record_attempt(self, request: ChatRequest, call_id: str, rec: AttemptRecord,
ledger: CostLedger | None, run: str = "",
- ctx: CallContext | None = None) -> bool:
+ ctx: CallContext | None = None, tallies: tuple[Tally, ...] | None = None) -> bool:
"""One row per round-trip. A round-trip that was DISCARDED and still cost money
(a billed-but-invalid reply, a hedge loser that landed) is money nothing else
records, so it goes in as ``source="extra"`` and counts in every total; the rest
@@ -150,13 +157,42 @@ class MeteredChatModel:
latency_ms=rec.usage.latency_ms,
source="extra" if extra else "attempt", key=_key_suffix(rec.key),
call_id=call_id, attempt=rec.attempt, discarded=rec.discarded,
- ledger=ledger)
+ ledger=ledger, tallies=tallies)
except Exception as e: # pragma: no cover - accounting must never break a call
log.debug("cost: could not record attempt %s#%d: %s", request.label, rec.attempt, e)
return False
return extra
+class _RoundTrips:
+ """The round-trips of ONE logical call, as the backend reported them through the attempt
+ sink (``models.retry.rotate_with_retries``), reduced to what the minutes need: how long
+ the ones that got an answer took, and whether any failed without one."""
+
+ def __init__(self) -> None:
+ self.answered_ms = 0 # the winner, and a reply that came back unusable (bad JSON, empty)
+ self.failed = False # a round-trip with no answer at all: 503 / 429 / timeout / transport / dead key
+ self.seen = False
+
+ def add(self, rec: AttemptRecord) -> None:
+ self.seen = True
+ if rec.discarded and rec.outcome == "ok":
+ return # a hedge loser that answered ran beside the winner, not before it
+ if rec.usage.latency_ms:
+ self.answered_ms += rec.usage.latency_ms
+ elif rec.outcome != "ok":
+ self.failed = True
+
+ def lost_ms(self, wall_ms: int, answer_ms: int) -> int:
+ """The call's milliseconds lost to provider errors: everything outside its answered
+ round-trips once one failed without an answer — its failed tries, their back-off and
+ the key waits between them. A backend that reports no round-trips (the SDK adapters)
+ is charged everything outside the answer it returned."""
+ if not self.seen:
+ return max(0, wall_ms - answer_ms)
+ return max(0, wall_ms - self.answered_ms) if self.failed else 0
+
+
def _key_suffix(key: object) -> str:
"""The last 4 chars of an API key as ``…ab12`` — the ledger never holds more."""
s = str(key or "").strip()
@@ -203,8 +239,11 @@ class MeteredAgent:
t0 = time.perf_counter()
with call_context(round=job.round, stage=stage, role=Role.GENERATOR, label=job.label):
result = self._inner.run(job)
+ wall_s = time.perf_counter() - t0
+ # the CLI's own retries of 503 / 429 / token-limit errors, read from its log
+ book_time(wall_s, float(result.provider_wait_s or 0.0))
# an opaque CLI: its session row is the only record
- self._record_session(job, result, stage, int((time.perf_counter() - t0) * 1000))
+ self._record_session(job, result, stage, int(wall_s * 1000))
return result
def _record_session(self, job: AgentJob, result: AgentResult, stage: Stage, ms: int) -> None:
diff --git a/harness/codeverse3d/cost/ledger.py b/harness/codeverse3d/cost/ledger.py
index 29e9fc72..d217ff61 100644
--- a/harness/codeverse3d/cost/ledger.py
+++ b/harness/codeverse3d/cost/ledger.py
@@ -27,6 +27,7 @@ from typing import Any
from codeverse3d.contracts.common import Usage
from codeverse3d.cost.context import CallContext, _stage, attribute
+from codeverse3d.cost.tally import Tally, book_usage
from codeverse3d.cost.types import (
CallCost,
CostBucket,
@@ -256,9 +257,12 @@ def record_call(
ledger: CostLedger | str | Path | None = None,
ts: float | None = None,
reprice: bool = False,
+ tallies: tuple[Tally, ...] | None = None,
**extra: Any,
) -> CallCost:
- """Append one priced call to the ledger and return the row.
+ """Append one priced call to the ledger, book it into the open tallies
+ (``cost.tally`` — or ``tallies``, captured by a caller whose row lands later in a
+ thread with no context), and return the row.
Callers pass the ``Usage`` they already hold::
@@ -309,6 +313,11 @@ def record_call(
target = ledger if isinstance(ledger, CostLedger) else (CostLedger(ledger) if ledger else default_ledger())
if target is not None:
target.append(row)
+ if source != "attempt": # an attempt row's tokens are already on its call's own row
+ book_usage(Usage(backend=row.backend, model=row.model, input_tokens=row.input_tokens,
+ output_tokens=row.output_tokens, cached_tokens=row.cached_tokens,
+ thoughts_tokens=row.thoughts_tokens, tool_calls=row.tool_calls,
+ cost_usd=row.cost_usd, latency_ms=row.latency_ms), tallies)
return row
diff --git a/harness/codeverse3d/cost/tally.py b/harness/codeverse3d/cost/tally.py
new file mode 100644
index 00000000..2951dd76
--- /dev/null
+++ b/harness/codeverse3d/cost/tally.py
@@ -0,0 +1,105 @@
+"""What the meters saw inside one block of work: its money and its lost time.
+
+A :class:`Tally` is opened with :func:`tally` around a block (a round, a timed step) and
+nests; everything metered inside the block books into every tally that is open:
+
+* ``cost.ledger.record_call`` books each priced row, so ``Tally.usage`` is exactly the
+ money ``telemetry/cost.jsonl`` holds for the block — the round's cost is read here, never
+ accumulated a second way;
+* the proxies in ``cost.instrument`` book each API call's and each agent session's wall
+ clock and the part of it lost to provider errors (503 / 429 / overloaded / timeout
+ retries and their back-off), per thread, so :meth:`Tally.lost_s` knows what the block
+ lost even when its work ran side by side.
+
+:func:`timed` turns one block into a ``StepTime`` row — the unit a run's minutes are
+summed from (owner, 2026-09-22). The binding is a ``ContextVar``: ``proc.fan_out`` copies
+the context, so a worker thread books into its caller's tallies.
+"""
+
+from __future__ import annotations
+
+import threading
+import time
+from collections.abc import Iterator
+from contextlib import contextmanager
+from contextvars import ContextVar
+
+from codeverse3d.contracts.common import Usage
+from codeverse3d.contracts.run import StepTime
+
+
+class Tally:
+ """The money and the lost seconds one block of work booked (see the module docstring)."""
+
+ def __init__(self) -> None:
+ self.usage = Usage()
+ self._time: dict[int, list[float]] = {} # thread id → [wall_s, lost_s] of its metered calls
+ self._lock = threading.Lock()
+
+ def add_usage(self, usage: Usage) -> None:
+ with self._lock:
+ self.usage = self.usage + usage
+
+ def add_time(self, wall_s: float, lost_s: float) -> None:
+ with self._lock:
+ t = self._time.setdefault(threading.get_ident(), [0.0, 0.0])
+ t[0] += wall_s
+ t[1] += min(lost_s, wall_s)
+
+ def lost_s(self) -> float:
+ """Seconds of the block's clock lost to provider errors. Calls in one thread ran one
+ after the other, so their losses add up; threads ran side by side (fan-out sessions,
+ judge samples, best-of-N candidates), so the block lost what its slowest thread would
+ have been spared: ``max(wall) - max(wall - lost)`` over the threads."""
+ with self._lock:
+ if not self._time:
+ return 0.0
+ return max(w for w, _ in self._time.values()) - max(w - lost for w, lost in self._time.values())
+
+
+_open: ContextVar[tuple[Tally, ...]] = ContextVar("c3d_cost_tallies", default=())
+
+
+@contextmanager
+def tally() -> Iterator[Tally]:
+ """Open a tally for the block (nested ones all receive what the block books)."""
+ t = Tally()
+ token = _open.set(_open.get() + (t,))
+ try:
+ yield t
+ finally:
+ _open.reset(token)
+
+
+def open_tallies() -> tuple[Tally, ...]:
+ """The tallies this execution context books into — captured by a caller whose work may
+ land later in a thread with no context (a hedge loser's billed round-trip)."""
+ return _open.get()
+
+
+def book_usage(usage: Usage, tallies: tuple[Tally, ...] | None = None) -> None:
+ for t in open_tallies() if tallies is None else tallies:
+ t.add_usage(usage)
+
+
+def book_time(wall_s: float, lost_s: float) -> None:
+ for t in open_tallies():
+ t.add_time(max(0.0, wall_s), max(0.0, lost_s))
+
+
+@contextmanager
+def timed(step: str, into: list[StepTime], *, round_index: int | None = None) -> Iterator[Tally]:
+ """Time ``step`` into ``into`` — its wall clock and what it lost — whether the block
+ returns or raises. A step that took no time (a cached stage) leaves no row."""
+ t0 = time.monotonic()
+ with tally() as t:
+ try:
+ yield t
+ finally:
+ wall = time.monotonic() - t0
+ if wall >= 0.01:
+ into.append(StepTime(step=step, round=round_index, wall_s=round(wall, 2),
+ lost_s=round(min(wall, t.lost_s()), 2)))
+
+
+__all__ = ["Tally", "book_time", "book_usage", "open_tallies", "tally", "timed"]
diff --git a/harness/codeverse3d/orchestrator.py b/harness/codeverse3d/orchestrator.py
index b65f0db9..eed26e4e 100644
--- a/harness/codeverse3d/orchestrator.py
+++ b/harness/codeverse3d/orchestrator.py
@@ -25,9 +25,10 @@ from pydantic import BaseModel, Field, ValidationError
from codeverse3d.contracts.artifacts import GateReport, Judgment
from codeverse3d.contracts.common import Budget, Usage
from codeverse3d.contracts.plan import AcceptanceItem, Plan
-from codeverse3d.contracts.run import RoundRecord, RunStatus
+from codeverse3d.contracts.run import RoundRecord, RunStatus, StepTime
from codeverse3d.conventions import to_snake
from codeverse3d.cost.billing import bills_usd
+from codeverse3d.cost.tally import timed
from codeverse3d.proc import EventLog
from codeverse3d.workspace import Workspace
@@ -55,6 +56,8 @@ class RunState(BaseModel):
status: RunStatus = RunStatus.PLANNING
stages: dict[str, StageState] = Field(default_factory=dict)
+ steps: list[StepTime] = Field(default_factory=list, description="run-level timed steps, every session's "
+ "(a stage, best-of-N, the part of an aborted round that ran) — the run's minutes")
materialized_for: str = Field(default="", description="agent kind the workspace was materialised for")
stop_reason: str = ""
error: str = ""
@@ -139,8 +142,11 @@ class StageRunner:
*,
inputs: Any = "",
model: type[BaseModel] | None = None,
+ timed_step: bool = True,
) -> T:
- """Run ``fn`` unless a cached result for the same ``inputs`` exists."""
+ """Run ``fn`` unless a cached result for the same ``inputs`` exists. A stage that runs
+ is a timed step of the run (``RunState.steps``) — unless ``timed_step=False``: a stage
+ that runs beside its siblings is timed once, as their group, by its caller."""
h = hash_inputs(inputs)
path = self.result_path(name)
prior = self.state.stages.get(name)
@@ -166,7 +172,11 @@ class StageRunner:
self.events.emit("stage.start", stage=name, inputs_hash=h)
t0 = time.time()
try:
- result = fn()
+ if timed_step:
+ with timed(name, self.state.steps):
+ result = fn()
+ else:
+ result = fn()
except Exception as e:
self.events.emit("stage.failed", stage=name, error=f"{type(e).__name__}: {e}",
duration_s=round(time.time() - t0, 2))
diff --git a/harness/codeverse3d/tracks/candidates.py b/harness/codeverse3d/tracks/candidates.py
index b5775d1a..f72fe629 100644
--- a/harness/codeverse3d/tracks/candidates.py
+++ b/harness/codeverse3d/tracks/candidates.py
@@ -29,6 +29,7 @@ from codeverse3d.contracts.artifacts import BuildResult, Measurement, RenderSet
from codeverse3d.contracts.common import Usage
from codeverse3d.contracts.run import RoundRecord
from codeverse3d.conventions import OBJECT_VIEWS_QUICK
+from codeverse3d.cost.tally import timed
from codeverse3d.orchestrator import BudgetExceeded, gate_error_count
from codeverse3d.proc import EventLog, fan_out
from codeverse3d.tracks.common import RunContext
@@ -72,16 +73,20 @@ def run_best_of_n(track: Any, ctx: RunContext, tasks: Sequence[GenerationTask],
duration_s=cand.duration_s, cost_usd=round(rec.usage.cost_usd, 4), commit=rec.commit[:10])
return cand
- results = fan_out(list(range(n)), _one, max_workers=ctx.settings.limits.max_parallel_agents, label="candidates",
- item_name=lambda k: f"c{k}")
+ # the candidates run side by side: one step of the run's minutes (each candidate round
+ # keeps its own steps in its sub-workspace)
+ with timed("candidates", ctx.state.steps, round_index=0):
+ results = fan_out(list(range(n)), _one, max_workers=ctx.settings.limits.max_parallel_agents,
+ label="candidates", item_name=lambda k: f"c{k}")
# one retry for candidates that crashed (transient 503s, parse failures) — not for budget stops
retry = [k for k, r in enumerate(results) if isinstance(r, Exception) and not isinstance(r, BudgetExceeded)]
if retry and ctx.budget.ok():
for k in retry:
ctx.events.emit("candidate.retry", candidate=k, error=f"{type(results[k]).__name__}: {results[k]}"[:300])
subs[k] = make_candidate_context(track, ctx, k)
- again = fan_out(retry, _one, max_workers=ctx.settings.limits.max_parallel_agents, label="candidates-retry",
- item_name=lambda k: f"c{k}")
+ with timed("candidates", ctx.state.steps, round_index=0):
+ again = fan_out(retry, _one, max_workers=ctx.settings.limits.max_parallel_agents, label="candidates-retry",
+ item_name=lambda k: f"c{k}")
for k, r in zip(retry, again, strict=True):
results[k] = r
records: list[CandidateRecord] = []
diff --git a/harness/codeverse3d/tracks/lifecycle.py b/harness/codeverse3d/tracks/lifecycle.py
index 6ce9a412..bf2dc4e0 100644
--- a/harness/codeverse3d/tracks/lifecycle.py
+++ b/harness/codeverse3d/tracks/lifecycle.py
@@ -682,6 +682,7 @@ class BaseTrack:
if a not in prior_aborted]
return RunRecord(
spec=ctx.spec, plan=ctx.plan, workspace=str(ctx.ws.root), status=status, rounds=rounds, total_usage=total,
+ steps=list(ctx.state.steps),
environment={"python": platform.python_version(), "host": platform.node(), "track": self.track.value,
"language": ctx.language.value, "generator": ctx.agent_id},
prompt_hashes={**prior_hashes, **dict(ctx.prompt_hashes)}, started_at=ctx.state.started_at,
diff --git a/harness/codeverse3d/tracks/scene.py b/harness/codeverse3d/tracks/scene.py
index edbb5169..d8cd93de 100644
--- a/harness/codeverse3d/tracks/scene.py
+++ b/harness/codeverse3d/tracks/scene.py
@@ -39,6 +39,7 @@ from codeverse3d.contracts.common import TRACK_INFO, Track
from codeverse3d.contracts.plan import Plan, ScenePlan, ZonePlan
from codeverse3d.contracts.run import RoundRecord
from codeverse3d.conventions import to_snake
+from codeverse3d.cost.tally import timed
from codeverse3d.judges.base import judged_subset
from codeverse3d.languages.scene_threejs import zone_file
from codeverse3d.orchestrator import StageRunner, TaskGroup, compact_instructions
@@ -213,8 +214,9 @@ class SceneTrack(BaseTrack):
ctx.extra["textures"] = runner.stage("textures", lambda: self._textures_stage(ctx), inputs={"plan": plan}) or {}
stage_fns: dict[str, Any] = {"assets": lambda: run_asset_stage(ctx),
"env": lambda: self._env_stage(ctx), "layouts": _layouts}
- results = fan_out(list(stage_fns.items()), lambda kv: runner.stage(kv[0], kv[1], inputs=key),
- max_workers=2, label="assets+env", item_name=lambda kv: kv[0])
+ with timed("+".join(stage_fns), ctx.state.steps): # side by side: ONE step of the run's minutes
+ results = fan_out(list(stage_fns.items()), lambda kv: runner.stage(kv[0], kv[1], inputs=key, timed_step=False),
+ max_workers=2, label="assets+env", item_name=lambda kv: kv[0])
staged = dict(zip(stage_fns, results, strict=True))
first_exc = next((r for r in results if isinstance(r, Exception)), None)
if first_exc is not None:
diff --git a/harness/codeverse3d/tracks/steps.py b/harness/codeverse3d/tracks/steps.py
index 2ae271c3..1dbaf6f1 100644
--- a/harness/codeverse3d/tracks/steps.py
+++ b/harness/codeverse3d/tracks/steps.py
@@ -37,7 +37,8 @@ from codeverse3d.contracts.artifacts import (
)
from codeverse3d.contracts.common import Usage
from codeverse3d.contracts.plan import AcceptanceItem, Plan
-from codeverse3d.contracts.run import RoundRecord
+from codeverse3d.contracts.run import RoundRecord, StepTime
+from codeverse3d.cost.tally import timed
from codeverse3d.judges.base import round_input
from codeverse3d.judges.rubrics import is_degraded
from codeverse3d.orchestrator import BudgetExceeded, usage_delta
@@ -115,13 +116,15 @@ def load_round_journal(ws: Workspace) -> list[RoundRecord]:
return out
-def run_generation_tasks(ctx: RunContext, tasks: Sequence[GenerationTask]) -> list[GenerationResult]:
+def run_generation_tasks(ctx: RunContext, tasks: Sequence[GenerationTask], *,
+ steps: list[StepTime] | None = None) -> list[GenerationResult]:
"""Run tasks (parallel when > 1). Raises ``RoundFailed`` when none succeeded.
``GenerationTask.phase`` sequences the round: tasks run in parallel WITHIN a
phase, phases in ascending order. Every task is phase 0 unless a track says
otherwise, so this is a no-op for every existing caller; per-part scoped
- generation uses it so the assembly session sees the part files first.
+ generation uses it so the assembly session sees the part files first. Each
+ phase is one ``generate`` step in ``steps``.
"""
if not tasks:
return []
@@ -132,7 +135,7 @@ def run_generation_tasks(ctx: RunContext, tasks: Sequence[GenerationTask]) -> li
for ph in phases:
picked = [(i, t) for i, t in enumerate(tasks) if t.phase == ph]
try:
- got = _run_phase(ctx, [t for _, t in picked])
+ got = _run_phase(ctx, [t for _, t in picked], steps)
except RoundFailed as e:
# a phase in which nothing succeeded is not automatically a dead round:
# the earlier phases may have written the parts. Record the failures
@@ -146,19 +149,22 @@ def run_generation_tasks(ctx: RunContext, tasks: Sequence[GenerationTask]) -> li
if not ok_any:
raise RoundFailed.of(results_seq, "no generation task succeeded")
return results_seq
- return _run_phase(ctx, list(tasks))
+ return _run_phase(ctx, list(tasks), steps)
-def _run_phase(ctx: RunContext, tasks: Sequence[GenerationTask]) -> list[GenerationResult]:
- """One parallel batch of generation tasks (the pre-phase behaviour, unchanged)."""
+def _run_phase(ctx: RunContext, tasks: Sequence[GenerationTask],
+ steps: list[StepTime] | None = None) -> list[GenerationResult]:
+ """One parallel batch of generation tasks — one ``generate`` step: its sessions run side
+ by side, so they share one clock."""
def _one(task: GenerationTask) -> GenerationResult:
return generate_for(ctx, task)
# fan_out also for ONE task: a crashing generator (503 storm, parse error) becomes a failed
# result → RoundFailed (a refine round then stops the run as no_change) instead of killing it.
- results = fan_out(list(tasks), _one, max_workers=ctx.settings.limits.max_parallel_agents,
- label="generate", item_name=lambda t: t.label)
+ with timed("generate", [] if steps is None else steps, round_index=tasks[0].round if tasks else None):
+ results = fan_out(list(tasks), _one, max_workers=ctx.settings.limits.max_parallel_agents,
+ label="generate", item_name=lambda t: t.label)
out: list[GenerationResult] = []
budget_stop: Exception | None = None
for task, r in zip(tasks, results, strict=True):
@@ -201,15 +207,16 @@ def run_round(
clay views): the two knobs a best-of-N candidate turns (``candidates.run_best_of_n``
calls it directly for each candidate in its sub-workspace)."""
mark = ctx.budget.mark()
+ steps: list[StepTime] = []
try:
return _run_round(ctx, index=index, kind=kind, tasks=tasks, pipeline=pipeline, instructions=instructions,
previous=previous, files_hint=files_hint, extra_usage=extra_usage,
- extra_notes=extra_notes)
+ extra_notes=extra_notes, steps=steps)
except BaseException as e:
# the round died half-way (budget stop, 503 storm, RoundFailed). Whatever it
# burned is already in the guard: report it so the round is not invisible.
burned = usage_delta(ctx.budget.spent, mark)
- record_aborted_round(ctx, index=index, kind=kind, usage=burned, error=f"{type(e).__name__}: {e}")
+ record_aborted_round(ctx, index=index, kind=kind, usage=burned, error=f"{type(e).__name__}: {e}", steps=steps)
raise
@@ -227,8 +234,10 @@ def _run_round(
extra_notes: Sequence[str] = (),
render: RenderFn | None = None,
geometry_views: bool = True,
+ steps: list[StepTime] | None = None,
) -> RoundRecord:
t0 = time.time()
+ steps = [] if steps is None else steps
ctx.events.emit("round.start", round=index, kind=kind, n_tasks=len(tasks))
rec = RoundRecord(index=index, kind=kind, agent_backend=ctx.agent_id, instructions=list(instructions),
started_at=datetime.now(UTC))
@@ -243,7 +252,7 @@ def _run_round(
# generated with a strictly smaller library than a --candidates 1 baseline (review 2026-08-29)
skill_kind = "baseline" if kind == "candidate" else kind
skills_hook.attach_for_round(ctx, index=index, kind=skill_kind)
- gens = run_generation_tasks(ctx, skills_hook.with_inlined_skill(ctx, tasks))
+ gens = run_generation_tasks(ctx, skills_hook.with_inlined_skill(ctx, tasks), steps=steps)
turns = 0
for g in gens:
usage = usage + g.usage
@@ -254,8 +263,9 @@ def _run_round(
cost["generate"] = round(sum(g.usage.cost_usd for g in gens), 6)
ctx.ws.commit(f"r{index:02d} {kind}: generated")
- outcome: RepairOutcome = build_with_repair(ctx, round_index=index, label=f"r{index:02d}_{kind}",
- files_hint=list(files_hint))
+ with timed("build", steps, round_index=index):
+ outcome: RepairOutcome = build_with_repair(ctx, round_index=index, label=f"r{index:02d}_{kind}",
+ files_hint=list(files_hint))
usage = usage + outcome.usage
if outcome.usage.cost_usd:
cost["repair"] = round(outcome.usage.cost_usd, 6)
@@ -268,10 +278,12 @@ def _run_round(
)
if outcome.build.ok:
- rec.measurement = pipeline.measure(ctx, outcome.build)
- gates.extend(pipeline.gates(ctx, index, outcome.build, rec.measurement))
+ with timed("gates", steps, round_index=index):
+ rec.measurement = pipeline.measure(ctx, outcome.build)
+ gates.extend(pipeline.gates(ctx, index, outcome.build, rec.measurement))
try:
- rec.renders = (render or pipeline.render)(ctx, index, outcome.build, rec.measurement)
+ with timed("render", steps, round_index=index):
+ rec.renders = (render or pipeline.render)(ctx, index, outcome.build, rec.measurement)
except RenderError as e:
# A render that times out (measured 2026-08-26, art_verify camera_tripod: the
# refine round built in 1.2 s, then render_glb.mjs hit its 330 s timeout with
@@ -284,7 +296,8 @@ def _run_round(
ctx.events.emit("render.failed", round=index, error=str(e)[:400])
post = getattr(pipeline, "post_render_gates", None)
if callable(post) and rec.renders is not None:
- gates.extend(post(ctx, index, rec.renders))
+ with timed("gates", steps, round_index=index):
+ gates.extend(post(ctx, index, rec.renders))
# the build's own reports (scene_probe + shader_preflight, gl_frames) after the
# track's: the judge's warning list is capped and reads them in this order
gates.extend(outcome.build.gates)
@@ -296,8 +309,9 @@ def _run_round(
notes.append(f"judge skipped ({skip})")
ctx.events.emit("judge.skipped", round=index, reason=skip)
else:
- rec.judgment, judge_usage = _judge(ctx, pipeline, index, outcome.build, gates, rec, previous, notes,
- geometry_views=geometry_views)
+ with timed("judge", steps, round_index=index):
+ rec.judgment, judge_usage = _judge(ctx, pipeline, index, outcome.build, gates, rec, previous, notes,
+ geometry_views=geometry_views)
if rec.judgment is not None or judge_usage.cost_usd:
# a degraded/crashed verdict was still PAID: fold its usage into the
# round's usage and cost["judge"] so rec.usage and the cost.round event
@@ -323,6 +337,7 @@ def _run_round(
keep_round_artifacts(ctx.ws, index)
rec.usage = usage
rec.duration_s = round(time.time() - t0, 2)
+ rec.steps = steps
rec.notes = "; ".join(notes)
ctx.ws.write_json(round_record_path(ctx, index), rec)
ctx.events.emit("round.done", round=index, kind=kind, commit=rec.commit[:10], score=rec.score,
@@ -400,16 +415,19 @@ def _waste_flag(*, build_ok: bool, judged: bool, aborted: str) -> tuple[bool, st
return False, ""
-def record_aborted_round(ctx: RunContext, *, index: int, kind: str, usage: Usage, error: str) -> None:
+def record_aborted_round(ctx: RunContext, *, index: int, kind: str, usage: Usage, error: str,
+ steps: Sequence[StepTime] = ()) -> None:
"""A round that raised half-way still reports what it burned.
Its record is written next to the round records as ``aborted_rNN.json`` (never
``rNN.json``: the round did not happen, and ``load_round_journal`` must not
- resume from it), a ``cost.round`` event is emitted, and the usage is remembered
- in ``ctx.extra`` so the run record can carry it. Accounting a stop must never
- raise on top of the stop that is already happening."""
+ resume from it), a ``cost.round`` event is emitted, the usage is remembered
+ in ``ctx.extra`` so the run record can carry it, and the steps it did run join the
+ run-level steps (``RunState.steps``): their minutes were spent. Accounting a stop
+ must never raise on top of the stop that is already happening."""
try:
- rec = RoundRecord(index=index, kind=kind, agent_backend=ctx.agent_id, usage=usage,
+ ctx.state.steps.extend(steps)
+ rec = RoundRecord(index=index, kind=kind, agent_backend=ctx.agent_id, usage=usage, steps=list(steps),
notes=f"aborted: {error}")
path = ctx.ws.root / "rounds" / f"aborted_r{index:02d}.json"
path.parent.mkdir(parents=True, exist_ok=True)
@@ -491,7 +509,9 @@ def rejudge_round(ctx: RunContext, pipeline: RoundPipeline, rec: RoundRecord, pr
return False
ctx.events.emit("judge.retry", round=rec.index)
notes: list[str] = [rec.notes] if rec.notes else []
- judgment, judge_usage = _judge(ctx, pipeline, rec.index, rec.build, list(rec.gates), rec, previous, notes)
+ # a round recorded before step timing keeps counting its own clock (RoundRecord.minutes)
+ with timed("judge", rec.steps if rec.steps is not None else [], round_index=rec.index):
+ judgment, judge_usage = _judge(ctx, pipeline, rec.index, rec.build, list(rec.gates), rec, previous, notes)
if judgment is None:
# the retry failed AGAIN — but it was still paid for: keep the money and the
# failure note on the persisted record so totals and resume can see them.
diff --git a/harness/docs/ARCHITECTURE.md b/harness/docs/ARCHITECTURE.md
index 4b53019c..ec948a7e 100644
--- a/harness/docs/ARCHITECTURE.md
+++ b/harness/docs/ARCHITECTURE.md
@@ -163,6 +163,7 @@ codeverse3d/
profiles.py (economy|balanced|quality; cli._common.resolve_dial is THE resolver)
billing.py (SUBSCRIPTION_BACKENDS/bills_usd — which backends take real dollars,
so the ledger bills real money and not list price; docs/COST.md §25)
+ tally.py (a block's money = its ledger rows, its lost seconds → StepTime; COST §31)
guard.py reconstruct.py (old runs)
judges/ base.py (JudgeInput; `round_input` + `plan_summary(plan, language)`, the ONE payload
builder and plan digest the in-run judge, `3dcode judge`, calibration, the texture
diff --git a/harness/tests/cost/test_tally.py b/harness/tests/cost/test_tally.py
new file mode 100644
index 00000000..8847d6ad
--- /dev/null
+++ b/harness/tests/cost/test_tally.py
@@ -0,0 +1,140 @@
+"""Tallies: a block's money is its ledger rows, its minutes are its clock minus provider errors."""
+
+from __future__ import annotations
+
+import threading
+import time
+from pathlib import Path
+
+import pytest
+
+from codeverse3d.contracts.agent import AgentJob, AgentResult
+from codeverse3d.contracts.chat import ChatMessage, ChatRequest, ChatResponse
+from codeverse3d.contracts.common import Usage
+from codeverse3d.contracts.run import StepTime
+from codeverse3d.cost.context import AttemptRecord, attempt_sink
+from codeverse3d.cost.instrument import MeteredAgent, MeteredChatModel, run_ledger
+from codeverse3d.cost.ledger import load_ledger, record_call
+from codeverse3d.cost.tally import Tally, tally, timed
+
+REQ = ChatRequest(messages=[ChatMessage.user("x")], label="judge:static_object_v1:r00:s0")
+
+
+def test_a_tally_holds_exactly_the_ledger_rows_booked_inside_it(tmp_path: Path):
+ u = Usage(backend="gemini", model="gemini-3.7-flash", input_tokens=1_000, output_tokens=10)
+ with run_ledger(tmp_path, run="r"), tally() as outer:
+ record_call(u, stage="plan")
+ with tally() as inner:
+ record_call(u, stage="judge")
+ record_call(u, stage="judge", source="attempt") # forensics: its call's row carries it
+ assert inner.usage.input_tokens == 1_000 and outer.usage.input_tokens == 2_000
+ assert outer.usage.cost_usd == pytest.approx(sum(r.cost_usd for r in load_ledger(tmp_path)))
+
+
+def _side_by_side(*calls: tuple[float, float]) -> Tally:
+ """Book one (wall, lost) per thread, every thread alive at once (idents are reused after exit)."""
+ t, gate = Tally(), threading.Barrier(len(calls))
+
+ def worker(wall: float, lost: float) -> None:
+ gate.wait()
+ t.add_time(wall, lost)
+ gate.wait()
+
+ threads = [threading.Thread(target=worker, args=c) for c in calls]
+ for th in threads:
+ th.start()
+ for th in threads:
+ th.join()
+ return t
+
+
+def test_lost_time_adds_up_in_a_thread_and_takes_the_slowest_thread_side_by_side():
+ t = Tally()
+ t.add_time(60.0, 10.0)
+ t.add_time(30.0, 5.0) # one thread, one call after the other
+ assert t.lost_s() == pytest.approx(15.0)
+ # A lost 100 of its 600 s; B took 550 without an error: the step would have ended at 550
+ assert _side_by_side((600.0, 100.0), (550.0, 0.0)).lost_s() == pytest.approx(50.0)
+ # B took 600 anyway: A's errors cost the step nothing
+ assert _side_by_side((600.0, 100.0), (600.0, 0.0)).lost_s() == pytest.approx(0.0)
+ # both sat out the same storm: the step lost it once, not twice
+ assert _side_by_side((600.0, 100.0), (560.0, 100.0)).lost_s() == pytest.approx(100.0)
+
+
+def test_a_timed_step_is_one_row_with_its_wall_and_what_it_lost():
+ steps: list[StepTime] = []
+ with timed("judge", steps, round_index=2) as t:
+ t.add_time(5.0, 2.0)
+ time.sleep(0.03)
+ (row,) = steps
+ assert row.step == "judge" and row.round == 2 and row.wall_s >= 0.03
+ assert row.lost_s == pytest.approx(row.wall_s), "lost is clipped to the step's own clock"
+ with timed("skeleton", steps):
+ pass
+ assert len(steps) == 1, "a step that took no time leaves no row"
+ with pytest.raises(RuntimeError), timed("build", steps):
+ time.sleep(0.02)
+ raise RuntimeError("the time was still spent")
+ assert steps[-1].step == "build"
+
+
+class SinkChat:
+ """A backend that reports its round-trips like ``GeminiModel`` does."""
+
+ provider, model, id = "gemini", "gemini-3.7-flash", "gemini:gemini-3.7-flash"
+
+ def __init__(self, trips: list[tuple[str, int, float]]):
+ self.trips = trips # (outcome, answered latency ms, seconds the round-trip holds)
+
+ def generate(self, request: ChatRequest) -> ChatResponse:
+ sink = attempt_sink()
+ last = len(self.trips)
+ for i, (outcome, latency, hold) in enumerate(self.trips, 1):
+ time.sleep(hold)
+ sink(AttemptRecord(attempt=i, key="k", outcome=outcome, discarded=i != last,
+ usage=Usage(model=self.model, latency_ms=latency)))
+ return ChatResponse(text="ok", usage=Usage(model=self.model, input_tokens=10, latency_ms=self.trips[-1][1]))
+
+
+def _lost(chat: object, tmp_path: Path) -> tuple[float, float]:
+ with run_ledger(tmp_path, run="r"), tally() as t:
+ MeteredChatModel(chat).generate(REQ)
+ (wall, lost), = t._time.values()
+ return wall, lost
+
+
+def test_a_clean_call_loses_nothing_and_a_503_before_the_answer_is_lost(tmp_path: Path):
+ wall, lost = _lost(SinkChat([("ok", 50, 0.05)]), tmp_path)
+ assert lost == 0.0 and wall >= 0.05
+ wall, lost = _lost(SinkChat([("5xx", 0, 0.10), ("ok", 40, 0.04)]), tmp_path)
+ assert lost == pytest.approx(wall - 0.040, abs=0.02) and lost >= 0.09
+ # a reply that came back unusable is the model's own time, not the provider's
+ wall, lost = _lost(SinkChat([("skip", 60, 0.06), ("ok", 40, 0.04)]), tmp_path)
+ assert lost == 0.0
+
+
+def test_a_backend_that_reports_no_round_trips_is_charged_everything_outside_its_answer(tmp_path: Path):
+ class Sdk:
+ provider, model, id = "openai", "gpt-5", "openai:gpt-5"
+
+ def generate(self, request: ChatRequest) -> ChatResponse:
+ time.sleep(0.08) # a retried attempt and its back-off, invisible from here
+ return ChatResponse(text="ok", usage=Usage(model="gpt-5", latency_ms=20))
+
+ wall, lost = _lost(Sdk(), tmp_path)
+ assert lost == pytest.approx(wall - 0.020, abs=0.01)
+
+
+def test_an_agent_session_books_what_its_cli_lost_to_the_provider(tmp_path: Path):
+ class Cli:
+ kind, model, id = "gemini-cli", "gemini-3.7-flash", "gemini-cli:gemini-3.7-flash"
+
+ def run(self, job: AgentJob) -> AgentResult:
+ time.sleep(0.02)
+ return AgentResult(ok=True, provider_wait_s=0.015, usage=Usage(backend="gemini-cli", input_tokens=5))
+
+ with run_ledger(tmp_path, run="r"), tally() as t:
+ MeteredAgent(Cli()).run(AgentJob(workspace=str(tmp_path), prompt="p", label="baseline", round=0))
+ (wall, lost), = t._time.values()
+ assert wall >= 0.02 and lost == pytest.approx(0.015)
+ assert t.usage.input_tokens == 5, "the session row is booked into the tally like any ledger row"
diff --git a/harness/tests/orchestrator_tracks/test_step_timing.py b/harness/tests/orchestrator_tracks/test_step_timing.py
new file mode 100644
index 00000000..19a07d84
--- /dev/null
+++ b/harness/tests/orchestrator_tracks/test_step_timing.py
@@ -0,0 +1,61 @@
+"""A run's minutes are the sum of its steps' clocks minus what provider errors cost them
+(owner, 2026-09-22) — THE number every reader reports."""
+
+from __future__ import annotations
+
+import time
+
+from codeverse3d.contracts.agent import AgentJob, AgentResult
+from codeverse3d.contracts.common import Language
+from codeverse3d.contracts.run import RoundRecord, RunRecord, StepTime
+from codeverse3d.cost.instrument import MeteredAgent
+from codeverse3d.orchestrator import RunState
+from codeverse3d.tracks.static_object import StaticObjectTrack
+from codeverse3d.workspace import Workspace
+from tests.orchestrator_tracks.conftest import make_spec
+from tests.orchestrator_tracks.fakes import (
+ FakeAgent,
+ FakeJudge,
+ FakeRuntime,
+ FakeServices,
+ _planner,
+)
+
+
+class StormyAgent(FakeAgent):
+ """A vendor CLI that spent part of every session retrying 503s."""
+
+ def run(self, job: AgentJob) -> AgentResult:
+ res = super().run(job)
+ time.sleep(0.12)
+ return res.model_copy(update={"provider_wait_s": 0.08})
+
+
+def test_every_step_is_timed_and_the_minutes_leave_out_the_provider_errors(tmp_path, chair_plan, settings):
+ agent = MeteredAgent(StormyAgent(lambda job, ws: {"src/model.py": f"import bpy # r{job.round}\n"}))
+ ws = Workspace(tmp_path / "runs" / "timed")
+ track = StaticObjectTrack(services=FakeServices(), judge=FakeJudge(scores=(0.5, 0.6)), agent=agent,
+ planner_model=_planner(chair_plan.model_dump(mode="json")), settings=settings,
+ runtime=FakeRuntime(Language.BLENDER))
+ rec = track.run(make_spec(language=Language.BLENDER, max_rounds=1), ws)
+ for r in rec.rounds:
+ (gen,) = [s for s in r.steps if s.step == "generate"]
+ assert gen.round == r.index and gen.wall_s >= 0.12 and abs(gen.lost_s - 0.08) < 0.005
+ assert rec.steps is not None and all(s.round is None for s in rec.steps)
+ total = sum(s.active_s for s in rec.steps) + sum(s.active_s for r in rec.rounds for s in r.steps)
+ assert abs(rec.minutes - total / 60) < 1e-9
+ assert rec.minutes < sum(r.duration_s for r in rec.rounds) / 60, "the 503 retries are not in the minutes"
+ # the step log survives on disk: the journal carries each round's, run_state the run's
+ assert RoundRecord.model_validate(ws.read_json(ws.root / "rounds" / "r00.json")).steps == rec.rounds[0].steps
+ assert RunState.load(ws).steps == rec.steps
+
+
+def test_a_record_from_before_step_timing_counts_its_own_clock():
+ old = RunRecord.model_validate({"spec": make_spec().model_dump(mode="json"), "workspace": "w",
+ "rounds": [{"index": 0, "kind": "baseline", "duration_s": 120.0}],
+ "extra": {"budget": {"elapsed_min": 7.5}}})
+ assert old.steps is None and old.minutes == 7.5 and old.rounds[0].minutes == 2.0
+ assert RunRecord.model_validate({"spec": make_spec().model_dump(mode="json"), "workspace": "w"}).minutes is None
+ # a run resumed across the change: new steps, and an old round that still counts its clock
+ mixed = old.model_copy(update={"steps": [StepTime(step="plan", wall_s=90.0, lost_s=30.0)]})
+ assert mixed.minutes == 1.0 + 2.0