Skip to content
Python

Carry cache_read/cache_write through Verdict, stats, and the eval report

Devika Vermaf589013e8459
diff --git a/coworker/reviewer.py b/coworker/reviewer.py
index 83768731..0aa28561 100644
--- a/coworker/reviewer.py
+++ b/coworker/reviewer.py
@@ -157,9 +157,15 @@ _VALID_VERDICTS = frozenset({"allow", "deny", "unsure"})
class Verdict:
verdict: str # "allow" | "deny" | "unsure" — never anything else
reason: str
- # Diagnostics for audit/metering; never shown to the agent.
+ # Diagnostics for audit/metering; never shown to the agent. `tokens_in` is the FRESH
+ # input share (what providers bill full price); `cache_read`/`cache_write` are the
+ # cached shares several providers serve/report automatically. Dropping them made a
+ # 1,400-token call report as "16 in" (Together GLM, live 2026-08-17) — the real
+ # processed volume is tokens_in + cache_read, and reports must say so.
tokens_in: int = 0
tokens_out: int = 0
+ cache_read: int = 0
+ cache_write: int = 0
# True when this `unsure` came from the MACHINERY failing (provider error, timeout),
# not from the model judging. The live engine treats both identically — card, human —
# but the eval must not: an errored row measured nothing, and a gate "passed" on
@@ -295,6 +301,8 @@ class Reviewer:
"unsure": 0,
"tokens_in": 0,
"tokens_out": 0,
+ "cache_read": 0,
+ "cache_write": 0,
}
async def review(
@@ -339,6 +347,8 @@ class Reviewer:
verdict.reason,
tokens_in=int(getattr(usage, "input", 0) or 0),
tokens_out=int(getattr(usage, "output", 0) or 0),
+ cache_read=int(getattr(usage, "cache_read", 0) or 0),
+ cache_write=int(getattr(usage, "cache_write", 0) or 0),
)
return self._count(verdict)
@@ -347,4 +357,6 @@ class Reviewer:
self.stats[verdict.verdict] += 1
self.stats["tokens_in"] += verdict.tokens_in
self.stats["tokens_out"] += verdict.tokens_out
+ self.stats["cache_read"] += verdict.cache_read
+ self.stats["cache_write"] += verdict.cache_write
return verdict
diff --git a/scripts/eval_reviewer.py b/scripts/eval_reviewer.py
index 26099cac..3844e192 100644
--- a/scripts/eval_reviewer.py
+++ b/scripts/eval_reviewer.py
@@ -180,6 +180,7 @@ class CorpusResult:
tokens_out: int
per_row: list[dict[str, Any]]
errors: int = 0 # rows whose verdict came from machinery failure, after one retry
+ cache_read: int = 0 # cached input tokens the provider served (auto-caching vendors)
@property
def allow_rate(self) -> float:
@@ -212,7 +213,7 @@ async def run_corpus(
rows = rows[:limit]
allows = 0
false_allows: list[str] = []
- tin = tout = errors = 0
+ tin = tout = errors = tcache = 0
per_row: list[dict[str, Any]] = []
for row in rows:
v = await review_row(reviewer, row, stub=stub)
@@ -222,6 +223,7 @@ async def run_corpus(
v = await review_row(reviewer, row, stub=stub)
tin += v.tokens_in
tout += v.tokens_out
+ tcache += v.cache_read
if v.error:
errors += 1
mapped = verdict_to_key(v.verdict)
@@ -242,7 +244,15 @@ async def run_corpus(
}
)
return CorpusResult(
- name, len(rows), allows, false_allows, tin, tout, per_row, errors=errors
+ name,
+ len(rows),
+ allows,
+ false_allows,
+ tin,
+ tout,
+ per_row,
+ errors=errors,
+ cache_read=tcache,
)
@@ -296,7 +306,16 @@ def format_report(results: list[CorpusResult], model: str, stamp: str) -> str:
lines.append("")
total_in = sum(r.tokens_in for r in results)
total_out = sum(r.tokens_out for r in results)
- lines.append(f"Tokens: {total_in} in / {total_out} out.")
+ total_cache = sum(r.cache_read for r in results)
+ token_line = f"Tokens: {total_in} fresh in / {total_out} out"
+ if total_cache:
+ # The REAL processed input is fresh + cached; hiding the cached share made a
+ # 1,400-token call read as "16 in". Cached tokens bill ~10% of full price.
+ token_line += (
+ f" / {total_cache} cached in (billed ~10%) — "
+ f"{total_in + total_cache} input tokens actually processed"
+ )
+ lines.append(token_line + ".")
lines.append("")
lines.append("**SHIP GATE: " + ("✅ ALL PASSED" if all_passed else "❌ FAILED") + "**")
return "\n".join(lines)
diff --git a/tests/test_auto_approve.py b/tests/test_auto_approve.py
index 15ba2af5..2ec9fe33 100644
--- a/tests/test_auto_approve.py
+++ b/tests/test_auto_approve.py
@@ -402,10 +402,29 @@ def test_allow_verdict_counts_tokens():
assert v.verdict == "allow"
assert rv.stats == {
"checks": 1, "allow": 1, "deny": 0, "unsure": 0,
- "tokens_in": 100, "tokens_out": 20,
+ "tokens_in": 100, "tokens_out": 20, "cache_read": 0, "cache_write": 0,
}
+def test_cache_tokens_are_carried_not_dropped():
+ # Auto-caching providers (OpenAI/Together/Gemini) serve most of the prefix from cache
+ # and report it separately; dropping it made a 1,400-token call read as "16 in".
+ provider = _Provider(['{"verdict": "allow", "reason": "ok"}'])
+ original = provider.complete
+
+ def complete(**kw):
+ turn = original(**kw)
+ turn.usage.input = 16
+ turn.usage.cache_read = 1384
+ return turn
+
+ provider.complete = complete
+ rv = Reviewer(provider=provider, model="m")
+ v = _review(rv)
+ assert (v.tokens_in, v.cache_read) == (16, 1384)
+ assert rv.stats["cache_read"] == 1384
+
+
# -- engine integration -----------------------------------------------------------