Python
fix(auto-approve): the decision reviewer's state limit fits Jev's 32k per-question limit (OPE-224)
Devika Vermadf25480d1035
diff --git a/coworker/decision_reviewer.py b/coworker/decision_reviewer.pyindex 981758b7..299a592a 100644--- a/coworker/decision_reviewer.py+++ b/coworker/decision_reviewer.py@@ -51,11 +51,17 @@ from typing import Any, Callable, Optionalfrom .reviewer import APPROVAL_CONTEXT_MAX_CHARS, Verdict-# Jev reads at most 32k tokens of state. With no tokenizer for it, chars/4 is the usual-# rough estimate, and the margin is wide rather than discovering the limit as a 4xx.+# TypeSafe's documented limits (docs.typesafe.ai/models.md): 32k tokens for the state plus any+# ONE question, and 64k for the state plus all questions in a request; over either is a 400+# `max_tokens_exceeded`. The per-question limit binds first: our largest Jev question is about+# 950 characters (~320 tokens) and all of them together about 13k characters. There is no+# tokenizer for Jev; measured Jev traffic averaged about 3.3 characters per token (2026-10-18),+# and dense JSON or non-English text runs lower, so the estimate is 3. The 2,000-token margin+# covers the largest question and the request's own JSON. Oversized states fail closed to a+# card before any call (review(), below) instead of surfacing as a vendor error.JEV_STATE_TOKEN_LIMIT = 32_000-_CHARS_PER_TOKEN_ESTIMATE = 4-_STATE_CHAR_BUDGET = (JEV_STATE_TOKEN_LIMIT - 2_000) * _CHARS_PER_TOKEN_ESTIMATE+_CHARS_PER_TOKEN_ESTIMATE = 3+_STATE_CHAR_BUDGET = (JEV_STATE_TOKEN_LIMIT - 2_000) * _CHARS_PER_TOKEN_ESTIMATE # 90,000 characters# The values the Auto-Approve "reviewer" setting can take. "session" is the session's own# model (coworker.reviewer.Reviewer), the default.diff --git a/tests/test_decision_reviewer.py b/tests/test_decision_reviewer.pyindex c61acd16..3af401ea 100644--- a/tests/test_decision_reviewer.py+++ b/tests/test_decision_reviewer.py@@ -214,6 +214,30 @@ def test_jev_missing_part_fails_closed():assert {cond, *nots} <= {q.name for q in JEV_QUESTIONS}+def test_state_budget_fits_jevs_per_question_limit():+ # TypeSafe: 32k tokens for the state plus any one question. At a conservative 3 characters+ # per token, the largest state we send plus the largest Jev question must stay under it.+ from coworker.decision_reviewer import JEV_STATE_TOKEN_LIMIT, _STATE_CHAR_BUDGET++ rec = _Recorder(_answers({}))+ _, _, body = JEV(api_key="k", transport=rec).build_questions_request({}, JEV_QUESTIONS)+ largest_question = max(len(json.dumps(q, ensure_ascii=False)) for q in body["questions"].values())+ assert (_STATE_CHAR_BUDGET + largest_question) / 3 <= JEV_STATE_TOKEN_LIMIT+ assert _STATE_CHAR_BUDGET == 90_000++[email protected]("backend", [DECISIONS, JEV])+def test_oversized_state_fails_closed_without_a_vendor_call(backend):+ from coworker.decision_reviewer import _STATE_CHAR_BUDGET++ rv, rec = _rv(backend)+ # Under the 64k approval-context cap, but the action's own arguments push the state over.+ v = _review(rv, tool_name="write_file",+ arguments={"path": "big.txt", "content": "x" * (_STATE_CHAR_BUDGET + 1)})+ assert v.verdict == "unsure" and v.error+ assert rec.bodies == []++def test_jev_examples_are_not_eval_rows():from scripts import eval_reviewer as ev