Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(cost): align GLM 5.2 pricing with the benchmark catalog; record judge calls in session stats (#325)
Lin Jia2dd84598ccc9
diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rsindex 633d43ed2..820a800b4 100644--- a/crates/switchyard-server/src/lib.rs+++ b/crates/switchyard-server/src/lib.rs@@ -382,9 +382,21 @@ async fn serve_until_shutdown(#[derive(Clone, Copy)]struct RequestStart(Instant);+/// Tier recorded for classifier and judge calls in the routing log, distinguishing+/// routing overhead from the routed tiers a session was served by.+const CLASSIFIER_TIER: &str = "classifier";+/// Maps routed call observations to backend stats, non-routed calls to classifier/judge/// stats, and records routing overhead once the algorithm run completes.-fn stats_observer(stats: StatsAccumulator) -> RunObserver {+///+/// Successful classifier/judge calls are also appended to `classifier_log` when one is+/// configured, so per-session routing snapshots account for judge token overhead. Routed+/// calls stay off this path: the served call is logged with its terminal usage in+/// [`usage_metrics::observe`], which would make a second append here a double count.+fn stats_observer(+ stats: StatsAccumulator,+ classifier_log: Option<(SharedRoutingLog, routing_log::RoutingLogContext)>,+) -> RunObserver {Arc::new(move |observation| match observation {RunObservation::LlmCall(call) => {let latency_ms = call.duration.as_secs_f64() * 1_000.0;@@ -395,6 +407,16 @@ fn stats_observer(stats: StatsAccumulator) -> RunObserver {stats.record_error(&call.selected_model, call.tier.as_deref());}} else if call.is_success {+ if let (Some((log, context)), Some(usage)) =+ (classifier_log.as_ref(), call.usage.as_ref())+ {+ log.append(+ context.clone(),+ &call.selected_model,+ Some(CLASSIFIER_TIER),+ usage,+ );+ }stats.record_classifier_success(call.selected_model,call.usage.as_ref().map(usage_metrics::token_usage),@@ -656,7 +678,10 @@ async fn handle_llm_request(Err(response) => return response,};let algorithm = Arc::clone(&route.algorithm);- let observer = stats_observer(state.stats.clone());+ let observer = stats_observer(+ state.stats.clone(),+ state.routing_log.clone().zip(routing_log_context.clone()),+ );let (trace, response) = match algorithm.run_observed(Context::default(), request, Some(observer)).await@@ -1266,11 +1291,53 @@ fn endpoint_listing(has_routing_log: bool) -> String {#[cfg(test)]mod tests {+ use libsy::LlmCallObservation;use tokio::io::{AsyncReadExt, AsyncWriteExt};use tokio::sync::{Notify, oneshot};use super::*;+ /// A successful judge call lands in the per-session routing snapshot under its+ /// model id with the classifier tier, while routed calls stay off the observer's+ /// log path — they are logged with terminal usage when the served response is+ /// observed, so an append here would double count them.+ #[test]+ fn stats_observer_logs_judge_calls_to_the_routing_log() {+ let dir = tempfile::tempdir().expect("temp dir");+ let log = SharedRoutingLog::new(dir.path().join("routing.jsonl")).expect("routing log");+ let mut headers = HeaderMap::new();+ headers.insert("proxy_x_session_id", "session-1".parse().expect("header"));+ let context = routing_log::RoutingLogContext::from_headers(&headers);+ let observer = stats_observer(StatsAccumulator::default(), Some((log.clone(), context)));++ let call = |model: &str, is_routed: bool| {+ RunObservation::LlmCall(LlmCallObservation {+ selected_model: model.to_string(),+ tier: None,+ is_routed,+ is_success: true,+ duration: Duration::from_millis(3),+ usage: Some(Usage {+ input_tokens: Some(100),+ output_tokens: Some(7),+ ..Usage::default()+ }),+ })+ };+ observer(call("judge-model", false));+ observer(call("routed-model", true));++ let snapshot = log+ .snapshot_session("session-1")+ .expect("read log")+ .expect("session recorded");+ let snapshot = serde_json::to_value(&snapshot).expect("serializable snapshot");+ assert_eq!(snapshot["models"]["judge-model"]["calls"], 1);+ assert_eq!(snapshot["models"]["judge-model"]["prompt_tokens"], 100);+ assert_eq!(snapshot["models"]["judge-model"]["completion_tokens"], 7);+ assert!(snapshot["models"].get("routed-model").is_none());+ }+#[derive(Clone)]struct ShutdownTestState {started: Arc<Notify>,diff --git a/crates/switchyard-server/src/routing_log.rs b/crates/switchyard-server/src/routing_log.rsindex c4a2c46c8..88156dc92 100644--- a/crates/switchyard-server/src/routing_log.rs+++ b/crates/switchyard-server/src/routing_log.rs@@ -96,6 +96,7 @@ pub(crate) fn snapshot(}/// Request headers retained until terminal usage is available.+#[derive(Clone)]pub(crate) struct RoutingLogContext {task: Option<String>,trial_id: Option<String>,diff --git a/docs/routing_algorithms/escalation_router_routing.md b/docs/routing_algorithms/escalation_router_routing.mdindex 5cffce27b..4d3073c05 100644--- a/docs/routing_algorithms/escalation_router_routing.md+++ b/docs/routing_algorithms/escalation_router_routing.md@@ -153,6 +153,11 @@ The snapshot reports per-model calls, tokens, latency, and cost for the strongand weak tiers. Judge calls are recorded in the classifier stats bucket, so theirtoken cost and latency remain visible as routing overhead.+When the server runs with a routing log, successful judge calls also appear in+per-session routing stats under the judge's model id, tagged with the+`classifier` tier — so per-session token accounting includes judge overhead+alongside the tiers the session was served by.+## When not to use escalation routing- **One-shot requests.** No trajectory to judge. Usediff --git a/switchyard/lib/cost_estimator.py b/switchyard/lib/cost_estimator.pyindex 5cdb14888..19d747384 100644--- a/switchyard/lib/cost_estimator.py+++ b/switchyard/lib/cost_estimator.py@@ -125,16 +125,17 @@ MODEL_PRICING: dict[str, ModelPriceData] = {),# Z.ai GLM-5.2 — REFERENCE pricing only. Switchyard reaches this model as# ``glm-5.2-fp8`` on a self-hosted vLLM deployment, which has no per-token- # billing at all: the real cost is GPU-hours. These rates are- # SiliconFlow's published GLM-5.2 serverless tier (verified 2026-07-27),- # carried so benchmark arms can be compared against commercial models on a- # common basis — "what this traffic would have cost at market rates", not- # what we are billed. Provider rates vary about 2x (OpenRouter lists- # $0.76/$2.42), so treat the absolute number as indicative, not exact.- # OpenAI wire format and no documented cache-write premium, so- # cache_write = input.+ # billing at all: the real cost is GPU-hours. These rates are an average of+ # OpenRouter provider rates captured in late July 2026, not a first-party+ # Z.ai quote. Keeping the same snapshot across benchmark calculations makes+ # estimates comparable; it represents "what this traffic would have cost at+ # market rates", not what NVIDIA is billed. The snapshot provides input,+ # output, and cache-read prices but no cache-creation price, and the Hub does+ # not report a separate cache-write bucket for this deployment. Use the base+ # input rate as the fallback so any reported cache-write tokens are not+ # accidentally zero-rated."glm-5.2-fp8": ModelPriceData(- input=1.40, output=4.40, cached=0.26, cache_write=1.40,+ input=0.805, output=2.53, cached=0.1495, cache_write=0.805,),# DeepSeek V4 Flash — official api-docs.deepseek.com standard list# price (post-promo). 284B total / 13B active, 1M-token context