Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(libsy): update escalation judge param defaults to benchmarked values (#201)
Lin Jia54ae66c6f121
diff --git a/switchyard/lib/processors/escalation_judge_request_processor.py b/switchyard/lib/processors/escalation_judge_request_processor.pyindex a271e4861..3cdf44f51 100644--- a/switchyard/lib/processors/escalation_judge_request_processor.py+++ b/switchyard/lib/processors/escalation_judge_request_processor.py@@ -123,10 +123,10 @@ class EscalationJudgeConfig(BaseModel):"""First conversation turn on which the judge runs. Earlier turns have notrajectory to judge and always route weak."""- escalate_confirmations: int = Field(default=1, ge=1)+ escalate_confirmations: int = Field(default=2, ge=1)"""Consecutive ``escalate`` verdicts required before the latch fires.- ``1`` (default) pins on the first escalate verdict. ``2`` requires the+ ``1`` pins on the first escalate verdict. ``2`` (default) requires thejudge to confirm on the next judged turn, filtering one-shot eagerverdicts (a single failed command misread as a pattern) while keepingrecall on real trouble, which by definition persists across turns. A@@ -143,12 +143,12 @@ class EscalationJudgeConfig(BaseModel):keeps *recurring* is trouble, even when a quiet turn separates theflare-ups."""- recent_turn_window: int = Field(default=14, ge=1)+ recent_turn_window: int = Field(default=28, ge=1)"""Trailing messages shown to the judge on top of the anchors. Wider thanthe classifier's default because loop detection needs to see the repeats:a cycle longer than the window is invisible."""- max_request_chars: int = Field(default=12_000, ge=1_000)+ max_request_chars: int = Field(default=18_000, ge=1_000)"""Cap on the assembled judge transcript. When exceeded, the *oldest*window messages are dropped first — for a trajectory judge the newestevidence is strictly the most valuable."""@@ -162,7 +162,7 @@ class EscalationJudgeConfig(BaseModel):"""Cap for the first user message — the task statement the judge needsfor drift detection, so it gets the most generous anchor budget."""- window_message_chars: int = Field(default=300, ge=50)+ window_message_chars: int = Field(default=500, ge=50)"""Per-message cap inside the trailing window. Error signatures andcommand shapes survive this easily; full file dumps do not need to."""diff --git a/switchyard/lib/profiles/escalation_router_config.py b/switchyard/lib/profiles/escalation_router_config.pyindex 1e63b95fc..ffb208715 100644--- a/switchyard/lib/profiles/escalation_router_config.py+++ b/switchyard/lib/profiles/escalation_router_config.py@@ -42,11 +42,14 @@ class EscalationRouterConfig(BaseModel):judge_min_turn: First conversation turn on which the judge runs.judge_escalate_confirmations: Consecutive escalate verdicts requiredbefore the strong latch fires (``1`` pins on the first verdict).+ Defaults to the benchmarked configuration, which reached an+ equal-or-better solve rate while latching roughly a third as+ often as ``1`` — the router's main cost lever.The confirmation streak is tracked per process: with multipleworkers and no session-sticky load balancing, verdicts for oneconversation can land on different workers and take longer to- accumulate. Keep ``1`` (the default) on multi-worker deployments,- or route conversations sticky to a worker.+ accumulate. Set ``1`` on multi-worker deployments, or route+ conversations sticky to a worker.judge_confirmation_window: Judged turns an escalate verdict stayslive for confirmation; ``N > 1`` tolerates up to ``N - 1``intervening declines (recurring intermittent trouble confirms).@@ -71,7 +74,8 @@ class EscalationRouterConfig(BaseModel):judge_system_prompt: Optional judge prompt override. ``None`` uses thebuilt-in prompt.judge_timeout_s: Per-call judge timeout (seconds); the judge fails- open to the weak tier at timeout.+ open to the weak tier at timeout. Defaults to the benchmarked+ value. A judge target's own ``timeout_secs`` wins over this.session_key_depth: ``0`` (default) keys conversations on system +first user message (the shared Rust session key). ``N > 0``extends the key with the first ``N`` post-first-user messages so@@ -104,16 +108,16 @@ class EscalationRouterConfig(BaseModel):judge: LlmTargetfallback_target_on_evict: strjudge_min_turn: int = Field(default=3, ge=1)- judge_escalate_confirmations: int = Field(default=1, ge=1)+ judge_escalate_confirmations: int = Field(default=2, ge=1)judge_confirmation_window: int = Field(default=1, ge=1)judge_disable_reasoning: bool = Truejudge_max_completion_tokens: int | None = Field(default=None, ge=16)judge_dump_verdicts: bool = False- judge_recent_turn_window: int = Field(default=14, ge=1)- judge_window_message_chars: int = Field(default=300, ge=50)- judge_max_request_chars: int = Field(default=12_000, ge=1_000)+ judge_recent_turn_window: int = Field(default=28, ge=1)+ judge_window_message_chars: int = Field(default=500, ge=50)+ judge_max_request_chars: int = Field(default=18_000, ge=1_000)judge_system_prompt: str | None = Field(default=None, min_length=1)- judge_timeout_s: float = Field(default=5.0, gt=0.0)+ judge_timeout_s: float = Field(default=30.0, gt=0.0)session_key_depth: int = Field(default=0, ge=0)tier_timeout_s: float | None = Field(default=DEFAULT_DETERMINISTIC_TIER_TIMEOUT_S,diff --git a/tests/test_escalation_judge_request_processor.py b/tests/test_escalation_judge_request_processor.pyindex d671bd93b..c951531e8 100644--- a/tests/test_escalation_judge_request_processor.py+++ b/tests/test_escalation_judge_request_processor.py@@ -99,7 +99,9 @@ async def test_no_escalate_routes_weak_without_pin() -> None:async def test_escalate_routes_strong_and_latches() -> None:- processor, fake, _ = _processor(_verdict_json(True))+ # Single-verdict latch, stated explicitly: the default now requires a confirmation,+ # which test_confirmations_require_consecutive_escalates covers.+ processor, fake, _ = _processor(_verdict_json(True), escalate_confirmations=1)ctx = ProxyContext()await processor.process(ctx, _request())@@ -279,7 +281,9 @@ async def test_judge_failure_recorded_as_classifier_error() -> None:async def test_markdown_fenced_verdict_is_parsed() -> None:- processor, _, _ = _processor("```json\n" + _verdict_json(True) + "\n```")+ processor, _, _ = _processor(+ "```json\n" + _verdict_json(True) + "\n```", escalate_confirmations=1,+ )ctx = ProxyContext()await processor.process(ctx, _request())@@ -425,6 +429,7 @@ async def test_escalation_latch_isolated_by_deep_key() -> None:affinity = SessionAffinity(enabled=True)processor, fake, _ = _processor(_verdict_json(True), affinity=affinity, session_key_depth=1,+ escalate_confirmations=1,)def _trial(first_response: str, extra_turns: int = 2) -> ChatRequest:diff --git a/tests/test_route_bundle.py b/tests/test_route_bundle.pyindex 3eef241ee..726ccc3eb 100644--- a/tests/test_route_bundle.py+++ b/tests/test_route_bundle.py@@ -603,11 +603,11 @@ class TestEscalationRouterRouteType:c for c in switchyard.iter_components()if isinstance(c, EscalationJudgeRequestProcessor))- assert judge._config.escalate_confirmations == 1- assert judge._config.window_message_chars == 300+ assert judge._config.escalate_confirmations == 2+ assert judge._config.window_message_chars == 500assert judge._config.dump_verdicts_to_stderr is Falseassert judge._config.max_completion_tokens == 128- assert judge._config.timeout_s == 5.0+ assert judge._config.timeout_s == 30.0assert judge._affinity._l2 is Nonedef test_invalid_value_is_a_one_line_config_error(self):