Python
Carry cache_read/cache_write through Verdict, stats, and the eval report
Devika Vermaf589013e8459
diff --git a/coworker/reviewer.py b/coworker/reviewer.pyindex 83768731..0aa28561 100644--- a/coworker/reviewer.py+++ b/coworker/reviewer.py@@ -157,9 +157,15 @@ _VALID_VERDICTS = frozenset({"allow", "deny", "unsure"})class Verdict:verdict: str # "allow" | "deny" | "unsure" — never anything elsereason: str- # Diagnostics for audit/metering; never shown to the agent.+ # Diagnostics for audit/metering; never shown to the agent. `tokens_in` is the FRESH+ # input share (what providers bill full price); `cache_read`/`cache_write` are the+ # cached shares several providers serve/report automatically. Dropping them made a+ # 1,400-token call report as "16 in" (Together GLM, live 2026-08-17) — the real+ # processed volume is tokens_in + cache_read, and reports must say so.tokens_in: int = 0tokens_out: int = 0+ cache_read: int = 0+ cache_write: int = 0# True when this `unsure` came from the MACHINERY failing (provider error, timeout),# not from the model judging. The live engine treats both identically — card, human —# but the eval must not: an errored row measured nothing, and a gate "passed" on@@ -295,6 +301,8 @@ class Reviewer:"unsure": 0,"tokens_in": 0,"tokens_out": 0,+ "cache_read": 0,+ "cache_write": 0,}async def review(@@ -339,6 +347,8 @@ class Reviewer:verdict.reason,tokens_in=int(getattr(usage, "input", 0) or 0),tokens_out=int(getattr(usage, "output", 0) or 0),+ cache_read=int(getattr(usage, "cache_read", 0) or 0),+ cache_write=int(getattr(usage, "cache_write", 0) or 0),)return self._count(verdict)@@ -347,4 +357,6 @@ class Reviewer:self.stats[verdict.verdict] += 1self.stats["tokens_in"] += verdict.tokens_inself.stats["tokens_out"] += verdict.tokens_out+ self.stats["cache_read"] += verdict.cache_read+ self.stats["cache_write"] += verdict.cache_writereturn verdictdiff --git a/scripts/eval_reviewer.py b/scripts/eval_reviewer.pyindex 26099cac..3844e192 100644--- a/scripts/eval_reviewer.py+++ b/scripts/eval_reviewer.py@@ -180,6 +180,7 @@ class CorpusResult:tokens_out: intper_row: list[dict[str, Any]]errors: int = 0 # rows whose verdict came from machinery failure, after one retry+ cache_read: int = 0 # cached input tokens the provider served (auto-caching vendors)@propertydef allow_rate(self) -> float:@@ -212,7 +213,7 @@ async def run_corpus(rows = rows[:limit]allows = 0false_allows: list[str] = []- tin = tout = errors = 0+ tin = tout = errors = tcache = 0per_row: list[dict[str, Any]] = []for row in rows:v = await review_row(reviewer, row, stub=stub)@@ -222,6 +223,7 @@ async def run_corpus(v = await review_row(reviewer, row, stub=stub)tin += v.tokens_intout += v.tokens_out+ tcache += v.cache_readif v.error:errors += 1mapped = verdict_to_key(v.verdict)@@ -242,7 +244,15 @@ async def run_corpus(})return CorpusResult(- name, len(rows), allows, false_allows, tin, tout, per_row, errors=errors+ name,+ len(rows),+ allows,+ false_allows,+ tin,+ tout,+ per_row,+ errors=errors,+ cache_read=tcache,)@@ -296,7 +306,16 @@ def format_report(results: list[CorpusResult], model: str, stamp: str) -> str:lines.append("")total_in = sum(r.tokens_in for r in results)total_out = sum(r.tokens_out for r in results)- lines.append(f"Tokens: {total_in} in / {total_out} out.")+ total_cache = sum(r.cache_read for r in results)+ token_line = f"Tokens: {total_in} fresh in / {total_out} out"+ if total_cache:+ # The REAL processed input is fresh + cached; hiding the cached share made a+ # 1,400-token call read as "16 in". Cached tokens bill ~10% of full price.+ token_line += (+ f" / {total_cache} cached in (billed ~10%) — "+ f"{total_in + total_cache} input tokens actually processed"+ )+ lines.append(token_line + ".")lines.append("")lines.append("**SHIP GATE: " + ("✅ ALL PASSED" if all_passed else "❌ FAILED") + "**")return "\n".join(lines)diff --git a/tests/test_auto_approve.py b/tests/test_auto_approve.pyindex 15ba2af5..2ec9fe33 100644--- a/tests/test_auto_approve.py+++ b/tests/test_auto_approve.py@@ -402,10 +402,29 @@ def test_allow_verdict_counts_tokens():assert v.verdict == "allow"assert rv.stats == {"checks": 1, "allow": 1, "deny": 0, "unsure": 0,- "tokens_in": 100, "tokens_out": 20,+ "tokens_in": 100, "tokens_out": 20, "cache_read": 0, "cache_write": 0,}+def test_cache_tokens_are_carried_not_dropped():+ # Auto-caching providers (OpenAI/Together/Gemini) serve most of the prefix from cache+ # and report it separately; dropping it made a 1,400-token call read as "16 in".+ provider = _Provider(['{"verdict": "allow", "reason": "ok"}'])+ original = provider.complete++ def complete(**kw):+ turn = original(**kw)+ turn.usage.input = 16+ turn.usage.cache_read = 1384+ return turn++ provider.complete = complete+ rv = Reviewer(provider=provider, model="m")+ v = _review(rv)+ assert (v.tokens_in, v.cache_read) == (16, 1384)+ assert rv.stats["cache_read"] == 1384++# -- engine integration -----------------------------------------------------------