Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(benchmark): net out retried attempts by trial id in task-level routing stats (#120)

ryan-lempkaee5e96decf91
diff --git a/benchmark/README.md b/benchmark/README.md
index 6ca511cb0..1dd64f6da 100644
--- a/benchmark/README.md
+++ b/benchmark/README.md
@@ -286,10 +286,16 @@ deterministic LLM-classifier routes, `server.classifier_prompts` records each ro
prompt, prompt SHA-256, `max_request_chars`, and `recent_turn_window` for reproducibility. Direct
runs mark routing stats as `not-requested`.
-Switchyard runs also write `routing_requests.jsonl` (one record per router request: task, session,
-selected model, tier, token usage) and roll it up into `routing_stats_by_task.json` at finalize.
-The task identity comes from the per-task proxy sidecar, which stamps `x-switchyard-intake-task`
-and a per-trial `proxy_x_session_id` on model-bound requests.
+Switchyard runs also write `routing_requests.jsonl` (one record per router request: task, trial,
+session, served model, tier, and the six token fields from the global schema) and roll it up into
+`routing_stats_by_task.json` at finalize. The per-task proxy sidecar stamps the identity headers on
+model-bound requests: `x-switchyard-intake-task`, `x-switchyard-trial-id` (Harbor's trial name), and
+a per-attempt `proxy_x_session_id`.
+
+Each task in `routing_stats_by_task.json` splits into `final` and `retries`. Requests are grouped by
+trial, and within a trial the latest attempt is the one Harbor kept (it retries sequentially and
+discards earlier attempts), so tokens spent on retried-away attempts land in `retries` and can be
+netted out of cost.
## Docker Image Notes
diff --git a/benchmark/closed_book_proxy/proxy/rewriter.py b/benchmark/closed_book_proxy/proxy/rewriter.py
index 86dfe559f..6f4e21448 100644
--- a/benchmark/closed_book_proxy/proxy/rewriter.py
+++ b/benchmark/closed_book_proxy/proxy/rewriter.py
@@ -44,6 +44,14 @@ STRIP_PATH_SUFFIXES = (
)
INTAKE_TASK_HEADER = "x-switchyard-intake-task"
SESSION_ID_HEADER = "proxy_x_session_id"
+TRIAL_ID_HEADER = "x-switchyard-trial-id"
+
+
+def _trial_id_from_env() -> str:
+ # Harbor sets HOST_AGENT_LOGS_PATH to <trials_dir>/<trial_name>/agent, passed
+ # into the proxy as SWITCHYARD_TRIAL_DIR; the trial name is that dir's parent.
+ trial_dir = os.environ.get("SWITCHYARD_TRIAL_DIR", "").strip()
+ return Path(trial_dir).parent.name if trial_dir else ""
def _truthy(value: str | None) -> bool:
@@ -67,8 +75,9 @@ class ClosedBookProxy:
)
self.allowed_hosts = self._load_allowed_hosts()
self.task_id = os.environ.get("SWITCHYARD_TASK_ID", "")
- # One id per proxy boot; the sidecar boots once per trial, so this
- # distinguishes trials of the same task.
+ self.trial_id = _trial_id_from_env()
+ # One id per proxy boot; the sidecar boots once per attempt, so this
+ # distinguishes retries of the same trial.
self.session_id = uuid.uuid4().hex
def _load_allowed_hosts(self) -> set[str]:
@@ -128,6 +137,8 @@ class ClosedBookProxy:
if self.task_id:
flow.request.headers[INTAKE_TASK_HEADER] = self.task_id
flow.request.headers[SESSION_ID_HEADER] = self.session_id
+ if self.trial_id:
+ flow.request.headers[TRIAL_ID_HEADER] = self.trial_id
if not self.closed_book:
return
if "application/json" not in flow.request.headers.get("content-type", ""):
diff --git a/benchmark/prepare_harbor_dataset.py b/benchmark/prepare_harbor_dataset.py
index f7c90afbc..fb30a8ce3 100644
--- a/benchmark/prepare_harbor_dataset.py
+++ b/benchmark/prepare_harbor_dataset.py
@@ -412,6 +412,7 @@ def _merge_compose(task_dir: Path, proxy_allowlist_hosts: tuple[str, ...]) -> di
"OPENAI_BASE_URL=${OPENAI_BASE_URL:-}",
"SWITCHYARD_BASE_URL=${SWITCHYARD_BASE_URL:-}",
f"SWITCHYARD_TASK_ID={task_dir.name}",
+ "SWITCHYARD_TRIAL_DIR=${HOST_AGENT_LOGS_PATH:-}",
"VERIFIER_PROXY_TOKEN=${SWITCHYARD_VERIFIER_PROXY_TOKEN:-}",
],
"healthcheck": {
diff --git a/benchmark/run_manifest.py b/benchmark/run_manifest.py
index 5f286530d..92879b5bf 100644
--- a/benchmark/run_manifest.py
+++ b/benchmark/run_manifest.py
@@ -223,13 +223,67 @@ def _copy_if_present(source: Path | None, dest: Path | None) -> str:
return "present"
+_TOKEN_FIELDS = (
+ "prompt_tokens",
+ "cached_tokens",
+ "cache_creation_tokens",
+ "completion_tokens",
+ "reasoning_tokens",
+ "total_tokens",
+)
+
+
+def _new_section() -> dict[str, Any]:
+ return {"calls": 0, "totals": dict.fromkeys(_TOKEN_FIELDS, 0), "_buckets": {}}
+
+
+def _accumulate(section: dict[str, Any], record: dict[str, Any]) -> None:
+ section["calls"] += 1
+ model = record.get("model") or "unknown"
+ tier = record.get("tier") or ""
+ bucket = section["_buckets"].setdefault(
+ (model, tier),
+ {"model": model, "tier": tier, "calls": 0, **dict.fromkeys(_TOKEN_FIELDS, 0)},
+ )
+ bucket["calls"] += 1
+ for field in _TOKEN_FIELDS:
+ value = record.get(field)
+ value = value if isinstance(value, int) else 0
+ bucket[field] += value
+ section["totals"][field] += value
+
+
+def _finalize_section(section: dict[str, Any]) -> None:
+ buckets = section.pop("_buckets")
+ section["models"] = [buckets[key] for key in sorted(buckets)]
+
+
+def _kept_sessions(records: list[dict[str, Any]]) -> set[str]:
+ """The kept attempt of each trial is its latest-timestamped session.
+
+ Harbor retries sequentially and keeps only the final attempt, so within a
+ trial the session with the newest timestamp is the one that survived.
+ Records without a ``trial_id`` are all treated as kept (no netting).
+ """
+ latest: dict[str, tuple[str, str]] = {}
+ for record in records:
+ trial = record.get("trial_id")
+ session = record.get("session_id")
+ if not trial or not session:
+ continue
+ ts = record.get("ts") or ""
+ if trial not in latest or ts > latest[trial][0]:
+ latest[trial] = (ts, session)
+ return {session for _, session in latest.values()}
+
+
def summarize_routing_log(log_path: Path) -> dict[str, Any]:
- """Aggregate a per-request routing JSONL log into per-task stats.
+ """Roll the routing log into per-task ``final`` and ``retries`` sections.
- Each task entry lists one bucket per (model, tier) pair so a model that
- served more than one tier is not merged into a single row.
+ Requests are grouped by trial; the latest attempt in each trial is ``final``
+ and earlier attempts are ``retries`` (see :func:`_kept_sessions`).
"""
- tasks: dict[str, dict[str, Any]] = {}
+ by_task: dict[str, list[dict[str, Any]]] = {}
total_requests = 0
for raw in log_path.read_text().splitlines():
raw = raw.strip()
@@ -242,32 +296,30 @@ def summarize_routing_log(log_path: Path) -> dict[str, Any]:
if not isinstance(record, dict):
continue
total_requests += 1
- task = record.get("task") or "unattributed"
- entry = tasks.setdefault(task, {"requests": 0, "sessions": [], "_buckets": {}})
- entry["requests"] += 1
- session = record.get("session_id")
- if session and session not in entry["sessions"]:
- entry["sessions"].append(session)
- model = record.get("model") or "unknown"
- tier = record.get("tier") or ""
- bucket = entry["_buckets"].setdefault(
- (model, tier),
- {
- "model": model,
- "tier": tier,
- "calls": 0,
- "prompt_tokens": 0,
- "completion_tokens": 0,
- "total_tokens": 0,
- },
- )
- bucket["calls"] += 1
- for key in ("prompt_tokens", "completion_tokens", "total_tokens"):
- value = record.get(key)
- bucket[key] += value if isinstance(value, int) else 0
- for entry in tasks.values():
- buckets = entry.pop("_buckets")
- entry["models"] = [buckets[key] for key in sorted(buckets)]
+ by_task.setdefault(record.get("task") or "unattributed", []).append(record)
+
+ tasks: dict[str, dict[str, Any]] = {}
+ for task, records in by_task.items():
+ kept = _kept_sessions(records)
+ entry = {
+ "requests": len(records),
+ "n_retries": 0,
+ "final": _new_section(),
+ "retries": _new_section(),
+ }
+ retry_sessions: set[str] = set()
+ for record in records:
+ session = record.get("session_id")
+ # A record is a retry only when it has a trial_id and its session was
+ # not the kept one; untagged records default to final.
+ is_retry = bool(record.get("trial_id")) and session not in kept
+ _accumulate(entry["retries" if is_retry else "final"], record)
+ if is_retry and session:
+ retry_sessions.add(session)
+ entry["n_retries"] = len(retry_sessions)
+ _finalize_section(entry["final"])
+ _finalize_section(entry["retries"])
+ tasks[task] = entry
return {"total_requests": total_requests, "tasks": tasks}
diff --git a/switchyard/lib/processors/routing_log_response_processor.py b/switchyard/lib/processors/routing_log_response_processor.py
index 006f34c9b..89b2101c8 100644
--- a/switchyard/lib/processors/routing_log_response_processor.py
+++ b/switchyard/lib/processors/routing_log_response_processor.py
@@ -18,7 +18,7 @@ from switchyard.lib.chat_response.streaming_response_accumulator import (
attach_final_response_callback,
)
from switchyard.lib.proxy_context import CTX_PROXY_ACTUAL_MODEL, ProxyContext
-from switchyard.lib.request_metadata import CTX_REQUEST_METADATA
+from switchyard.lib.request_metadata import CTX_PROFILE_REQUEST_HEADERS, CTX_REQUEST_METADATA
from switchyard_rust.core import ChatResponse
logger = logging.getLogger(__name__)
@@ -57,11 +57,16 @@ class RoutingLogResponseProcessor:
def _write_record(self, ctx: ProxyContext, served_model: str, response: ChatResponse) -> None:
metadata = ctx.metadata.get(CTX_REQUEST_METADATA)
+ headers = ctx.metadata.get(CTX_PROFILE_REQUEST_HEADERS) or {}
+ # Prefer the model id the backend actually served so buckets reconcile
+ # with the global routing_stats schema, which keys on the served id.
+ actual_model = _field(response.body, "model")
record = {
"ts": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z",
"task": getattr(getattr(metadata, "intake", None), "task", None),
+ "trial_id": headers.get("x-switchyard-trial-id"),
"session_id": getattr(metadata, "session_id", None),
- "model": served_model,
+ "model": actual_model if isinstance(actual_model, str) and actual_model else served_model,
"tier": ctx.metadata.get("_random_routing_tier", "") or (ctx.selected_target or ""),
**_usage_tokens(response.body),
}
@@ -74,20 +79,40 @@ class RoutingLogResponseProcessor:
def _usage_tokens(body: object) -> dict[str, int]:
- """Token counts from any native usage shape (OpenAI Chat/Responses, Anthropic)."""
+ """Six-field token breakdown matching the global routing-stats schema.
+
+ cached/cache_creation are subsets of prompt_tokens, reasoning a subset of
+ completion_tokens, total = prompt + completion.
+ """
usage = _field(body, "usage")
prompt = _int_field(usage, "prompt_tokens")
completion = _int_field(usage, "completion_tokens")
- if not prompt and not completion:
- prompt = (
- _int_field(usage, "input_tokens")
- + _int_field(usage, "cache_creation_input_tokens")
- + _int_field(usage, "cache_read_input_tokens")
- )
+ cached = 0
+ cache_creation = 0
+ reasoning = 0
+ if prompt or completion:
+ # OpenAI Chat Completions.
+ cached = _int_field(_field(usage, "prompt_tokens_details"), "cached_tokens")
+ reasoning = _int_field(_field(usage, "completion_tokens_details"), "reasoning_tokens")
+ else:
completion = _int_field(usage, "output_tokens")
+ reasoning = _int_field(_field(usage, "output_tokens_details"), "reasoning_tokens")
+ input_details = _field(usage, "input_tokens_details")
+ if input_details is not None:
+ # OpenAI Responses API: prompt_tokens already includes cached.
+ prompt = _int_field(usage, "input_tokens")
+ cached = _int_field(input_details, "cached_tokens")
+ else:
+ # Anthropic Messages: cache tokens are prompt siblings, so fold in.
+ cached = _int_field(usage, "cache_read_input_tokens")
+ cache_creation = _int_field(usage, "cache_creation_input_tokens")
+ prompt = _int_field(usage, "input_tokens") + cached + cache_creation
return {
"prompt_tokens": prompt,
+ "cached_tokens": cached,
+ "cache_creation_tokens": cache_creation,
"completion_tokens": completion,
+ "reasoning_tokens": reasoning,
"total_tokens": prompt + completion,
}
diff --git a/tests/test_prepare_harbor_dataset.py b/tests/test_prepare_harbor_dataset.py
index 37f6a55be..1c69052ff 100644
--- a/tests/test_prepare_harbor_dataset.py
+++ b/tests/test_prepare_harbor_dataset.py
@@ -305,3 +305,4 @@ def test_generated_compose_bakes_task_id_into_proxy_env(tmp_path: Path) -> None:
proxy_env = "\n".join(compose["services"]["proxy"]["environment"])
assert "SWITCHYARD_TASK_ID=task-id-check" in proxy_env
+ assert "SWITCHYARD_TRIAL_DIR=${HOST_AGENT_LOGS_PATH:-}" in proxy_env
diff --git a/tests/test_routing_log_response_processor.py b/tests/test_routing_log_response_processor.py
index c404a84fb..62af541be 100644
--- a/tests/test_routing_log_response_processor.py
+++ b/tests/test_routing_log_response_processor.py
@@ -26,6 +26,7 @@ from switchyard_rust.core import ChatResponse, ProxyContext
TASK_HEADERS = {
"x-switchyard-intake-task": "hello-world-abc1",
+ "x-switchyard-trial-id": "hello-world-abc1-Xy7",
"proxy_x_session_id": "trial-session-1",
}
@@ -49,7 +50,13 @@ def _openai_completion() -> ChatResponse:
"message": {"role": "assistant", "content": "hi"},
"finish_reason": "stop",
}],
- "usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8},
+ "usage": {
+ "prompt_tokens": 8,
+ "completion_tokens": 3,
+ "total_tokens": 11,
+ "prompt_tokens_details": {"cached_tokens": 6},
+ "completion_tokens_details": {"reasoning_tokens": 2},
+ },
})
@@ -87,7 +94,13 @@ def _responses_completion() -> ChatResponse:
"parallel_tool_calls": False,
"tool_choice": "auto",
"tools": [],
- "usage": {"input_tokens": 6, "output_tokens": 2, "total_tokens": 8},
+ "usage": {
+ "input_tokens": 6,
+ "output_tokens": 2,
+ "total_tokens": 8,
+ "input_tokens_details": {"cached_tokens": 3},
+ "output_tokens_details": {"reasoning_tokens": 1},
+ },
})
@@ -103,12 +116,16 @@ async def test_openai_completion_record_carries_task_and_usage(tmp_path: Path) -
(record,) = _records(log_file)
assert record["task"] == "hello-world-abc1"
+ assert record["trial_id"] == "hello-world-abc1-Xy7"
assert record["session_id"] == "trial-session-1"
assert record["model"] == "gpt-test"
assert record["tier"] == "weak"
- assert record["prompt_tokens"] == 5
+ assert record["prompt_tokens"] == 8
+ assert record["cached_tokens"] == 6
+ assert record["cache_creation_tokens"] == 0
assert record["completion_tokens"] == 3
- assert record["total_tokens"] == 8
+ assert record["reasoning_tokens"] == 2
+ assert record["total_tokens"] == 11
async def test_anthropic_usage_sums_cache_siblings(tmp_path: Path) -> None:
@@ -119,7 +136,10 @@ async def test_anthropic_usage_sums_cache_siblings(tmp_path: Path) -> None:
(record,) = _records(log_file)
assert record["prompt_tokens"] == 13 # input + cache_creation + cache_read
+ assert record["cached_tokens"] == 4
+ assert record["cache_creation_tokens"] == 2
assert record["completion_tokens"] == 3
+ assert record["reasoning_tokens"] == 0
async def test_responses_completion_uses_input_output_tokens(tmp_path: Path) -> None:
@@ -130,7 +150,10 @@ async def test_responses_completion_uses_input_output_tokens(tmp_path: Path) ->
(record,) = _records(log_file)
assert record["prompt_tokens"] == 6
+ assert record["cached_tokens"] == 3
+ assert record["cache_creation_tokens"] == 0
assert record["completion_tokens"] == 2
+ assert record["reasoning_tokens"] == 1
async def test_missing_headers_log_null_task_and_session(tmp_path: Path) -> None:
@@ -139,6 +162,7 @@ async def test_missing_headers_log_null_task_and_session(tmp_path: Path) -> None
(record,) = _records(log_file)
assert record["task"] is None
+ assert record["trial_id"] is None
assert record["session_id"] is None
diff --git a/tests/test_run_manifest.py b/tests/test_run_manifest.py
index a26d50215..18a6a5513 100644
--- a/tests/test_run_manifest.py
+++ b/tests/test_run_manifest.py
@@ -469,59 +469,107 @@ def test_finalize_copies_proxy_strip_log_artifact(tmp_path: Path) -> None:
assert (run_dir / "proxy_strip_log.jsonl").read_text() == '{"removed":["web_search"]}\n'
+def _record(task: str, ts: str, model: str, tier: str, *, trial: str = "t", session: str = "s",
+ **tokens: int) -> str:
+ base = {"prompt_tokens": 0, "cached_tokens": 0, "cache_creation_tokens": 0,
+ "completion_tokens": 0, "reasoning_tokens": 0, "total_tokens": 0}
+ base.update(tokens)
+ return json.dumps({"task": task, "ts": ts, "trial_id": trial, "session_id": session,
+ "model": model, "tier": tier, **base})
+
+
def test_summarize_routing_log_groups_by_task_and_model(tmp_path: Path) -> None:
module = _load_manifest_module()
log = tmp_path / "routing_requests.jsonl"
log.write_text(
"\n".join([
- json.dumps({"task": "task-a", "session_id": "s1", "model": "opus",
- "tier": "strong", "prompt_tokens": 10, "completion_tokens": 2,
- "total_tokens": 12}),
- json.dumps({"task": "task-a", "session_id": "s1", "model": "kimi",
- "tier": "weak", "prompt_tokens": 4, "completion_tokens": 1,
- "total_tokens": 5}),
- json.dumps({"task": "task-b", "session_id": "s2", "model": "opus",
- "tier": "strong", "prompt_tokens": 7, "completion_tokens": 3,
- "total_tokens": 10}),
+ _record("task-a", "2026-01-01T00:00:01Z", "opus", "strong",
+ prompt_tokens=10, cached_tokens=6, completion_tokens=2,
+ reasoning_tokens=1, total_tokens=12),
+ _record("task-a", "2026-01-01T00:00:02Z", "kimi", "weak",
+ prompt_tokens=4, completion_tokens=1, total_tokens=5),
"not json",
- json.dumps({"model": "opus", "prompt_tokens": 1, "completion_tokens": 1,
- "total_tokens": 2}),
+ json.dumps(["not", "a", "dict"]),
]) + "\n"
)
summary = module.summarize_routing_log(log)
- assert summary["total_requests"] == 4
+ assert summary["total_requests"] == 2
task_a = summary["tasks"]["task-a"]
assert task_a["requests"] == 2
- assert task_a["sessions"] == ["s1"]
- assert task_a["models"] == [
- {"model": "kimi", "tier": "weak", "calls": 1, "prompt_tokens": 4,
- "completion_tokens": 1, "total_tokens": 5},
- {"model": "opus", "tier": "strong", "calls": 1, "prompt_tokens": 10,
- "completion_tokens": 2, "total_tokens": 12},
+ assert task_a["n_retries"] == 0
+ assert task_a["retries"]["calls"] == 0
+ assert task_a["final"]["calls"] == 2
+ assert task_a["final"]["totals"] == {
+ "prompt_tokens": 14, "cached_tokens": 6, "cache_creation_tokens": 0,
+ "completion_tokens": 3, "reasoning_tokens": 1, "total_tokens": 17,
+ }
+ assert [(b["model"], b["tier"], b["calls"]) for b in task_a["final"]["models"]] == [
+ ("kimi", "weak", 1), ("opus", "strong", 1),
]
- assert summary["tasks"]["task-b"]["models"][0]["total_tokens"] == 10
- assert summary["tasks"]["unattributed"]["requests"] == 1
-def test_summarize_routing_log_keeps_mixed_tiers_separate(tmp_path: Path) -> None:
+def test_summarize_routing_log_nets_out_retried_attempt(tmp_path: Path) -> None:
module = _load_manifest_module()
log = tmp_path / "routing_requests.jsonl"
+ # Same trial, two attempts: the earlier session was retried away.
log.write_text(
"\n".join([
- json.dumps({"task": "task-a", "model": "opus", "tier": "strong",
- "prompt_tokens": 10, "completion_tokens": 2, "total_tokens": 12}),
- json.dumps({"task": "task-a", "model": "opus", "tier": "weak",
- "prompt_tokens": 4, "completion_tokens": 1, "total_tokens": 5}),
+ _record("task-a", "2026-01-01T00:00:01Z", "opus", "strong",
+ trial="trial-1", session="retry-sess",
+ prompt_tokens=100, completion_tokens=10, total_tokens=110),
+ _record("task-a", "2026-01-01T00:05:01Z", "opus", "strong",
+ trial="trial-1", session="final-sess",
+ prompt_tokens=20, completion_tokens=3, total_tokens=23),
]) + "\n"
)
- buckets = module.summarize_routing_log(log)["tasks"]["task-a"]["models"]
+ task_a = module.summarize_routing_log(log)["tasks"]["task-a"]
- assert [(b["model"], b["tier"], b["calls"]) for b in buckets] == [
- ("opus", "strong", 1), ("opus", "weak", 1),
- ]
+ assert task_a["n_retries"] == 1
+ assert task_a["final"]["totals"]["total_tokens"] == 23
+ assert task_a["retries"]["totals"]["total_tokens"] == 110
+ assert task_a["final"]["calls"] == 1
+ assert task_a["retries"]["calls"] == 1
+
+
+def test_summarize_routing_log_keeps_all_concurrent_trials(tmp_path: Path) -> None:
+ module = _load_manifest_module()
+ log = tmp_path / "routing_requests.jsonl"
+ # Two trials of the same task (k>1). Neither retried, so both are final even
+ # though their calls interleave in time.
+ log.write_text(
+ "\n".join([
+ _record("task-a", "2026-01-01T00:00:01Z", "opus", "strong",
+ trial="trial-1", session="s1", total_tokens=10),
+ _record("task-a", "2026-01-01T00:00:02Z", "opus", "strong",
+ trial="trial-2", session="s2", total_tokens=20),
+ ]) + "\n"
+ )
+
+ task_a = module.summarize_routing_log(log)["tasks"]["task-a"]
+
+ assert task_a["n_retries"] == 0
+ assert task_a["final"]["calls"] == 2
+ assert task_a["final"]["totals"]["total_tokens"] == 30
+ assert task_a["retries"]["calls"] == 0
+
+
+def test_summarize_routing_log_untagged_records_all_final(tmp_path: Path) -> None:
+ module = _load_manifest_module()
+ log = tmp_path / "routing_requests.jsonl"
+ # No trial_id -> no netting, everything counts as final.
+ log.write_text(json.dumps({
+ "task": "task-a", "ts": "2026-01-01T00:00:01Z", "session_id": "s1",
+ "model": "opus", "tier": "strong", "total_tokens": 5,
+ }) + "\n")
+
+ task_a = module.summarize_routing_log(log)["tasks"]["task-a"]
+
+ assert task_a["final"]["calls"] == 1
+ assert task_a["retries"]["calls"] == 0
+ assert task_a["n_retries"] == 0
def test_finalize_writes_routing_stats_by_task(tmp_path: Path) -> None:
@@ -543,7 +591,7 @@ def test_finalize_writes_routing_stats_by_task(tmp_path: Path) -> None:
assert rc == 0
summary = json.loads(by_task.read_text())
- assert summary["tasks"]["task-a"]["models"][0]["calls"] == 1
+ assert summary["tasks"]["task-a"]["final"]["models"][0]["calls"] == 1
manifest = json.loads(out.read_text())
assert manifest["outcomes"]["routing_stats_by_task_json_status"] == "present"