Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(stats): track max observed context tokens per model (#24)
Aloys Bailletbacab4cf617a
diff --git a/crates/switchyard-components/src/stats/accumulator.rs b/crates/switchyard-components/src/stats/accumulator.rsindex ee2f72602..61a99f835 100644--- a/crates/switchyard-components/src/stats/accumulator.rs+++ b/crates/switchyard-components/src/stats/accumulator.rs@@ -173,6 +173,9 @@ impl StatsAccumulator {{let stats = inner.model_stats_mut(model.clone());stats.prompt_tokens = stats.prompt_tokens.saturating_add(usage.prompt_tokens);+ stats.max_observed_context_tokens = stats+ .max_observed_context_tokens+ .max(usage.prompt_tokens.saturating_add(usage.completion_tokens));stats.completion_tokens = stats.completion_tokens.saturating_add(usage.completion_tokens);@@ -242,6 +245,9 @@ impl StatsAccumulator {let stats = inner.classifier_stats_mut(model.into());stats.calls = stats.calls.saturating_add(1);stats.prompt_tokens = stats.prompt_tokens.saturating_add(usage.prompt_tokens);+ stats.max_observed_context_tokens = stats+ .max_observed_context_tokens+ .max(usage.prompt_tokens.saturating_add(usage.completion_tokens));stats.completion_tokens = stats.completion_tokens.saturating_add(usage.completion_tokens);@@ -316,6 +322,9 @@ impl StatsAccumulator {let stats = inner.planner_stats_mut(model.into());stats.calls = stats.calls.saturating_add(1);stats.prompt_tokens = stats.prompt_tokens.saturating_add(usage.prompt_tokens);+ stats.max_observed_context_tokens = stats+ .max_observed_context_tokens+ .max(usage.prompt_tokens.saturating_add(usage.completion_tokens));stats.completion_tokens = stats.completion_tokens.saturating_add(usage.completion_tokens);@@ -554,6 +563,7 @@ fn build_model_snapshots(errors: stats.errors,request_pct,prompt_tokens: stats.prompt_tokens,+ max_observed_context_tokens: stats.max_observed_context_tokens,completion_tokens: stats.completion_tokens,total_tokens: token_total,token_pct,@@ -610,6 +620,7 @@ struct ModelStats {calls: u64,errors: u64,prompt_tokens: u64,+ max_observed_context_tokens: u64,completion_tokens: u64,cached_tokens: u64,cache_creation_tokens: u64,@@ -796,6 +807,8 @@ pub struct ModelStatsSnapshot {pub errors: u64,pub request_pct: f64,pub prompt_tokens: u64,+ /// Largest prompt-plus-completion token count observed in one completed response.+ pub max_observed_context_tokens: u64,pub completion_tokens: u64,pub total_tokens: u64,pub token_pct: f64,diff --git a/crates/switchyard-components/tests/stats_accumulator.rs b/crates/switchyard-components/tests/stats_accumulator.rsindex 9b005386d..6ba105904 100644--- a/crates/switchyard-components/tests/stats_accumulator.rs+++ b/crates/switchyard-components/tests/stats_accumulator.rs@@ -87,6 +87,7 @@ fn accumulator_matches_python_two_tier_snapshot_contract() -> Result<()> {assert_eq!(strong.tier.as_deref(), Some("strong"));assert_eq!(strong.request_pct, 50.0);assert_eq!(strong.token_pct, 73.53);+ assert_eq!(strong.max_observed_context_tokens, 125);assert_eq!(strong.avg_prompt_tokens, 100.0);assert_eq!(strong.avg_completion_tokens, 25.0);assert_eq!(strong.cache_hit_rate, 0.1);@@ -110,6 +111,32 @@ fn accumulator_matches_python_two_tier_snapshot_contract() -> Result<()> {Ok(())}+#[test]+fn accumulator_tracks_max_observed_context_tokens_per_model() -> Result<()> {+ let accumulator = StatsAccumulator::new();+ for (prompt_tokens, completion_tokens) in [(100, 10), (90, 50), (120, 5)] {+ accumulator.record_success("model-a", None, None)?;+ accumulator.record_usage(+ "model-a",+ TokenUsage {+ prompt_tokens,+ completion_tokens,+ ..TokenUsage::default()+ },+ None,+ None,+ None,+ )?;+ }++ let snapshot = accumulator.snapshot()?;+ let model = model_stats(&snapshot, "model-a")?;+ assert_eq!(model.prompt_tokens, 310);+ assert_eq!(model.completion_tokens, 65);+ assert_eq!(model.max_observed_context_tokens, 140);+ Ok(())+}+// Switch-aware theoretical: cold on switch-in, partial credit on switching back.#[test]fn theoretical_is_switch_aware_across_a_model_switch_and_back() -> Result<()> {@@ -653,6 +680,7 @@ fn classifier_bucket_keeps_same_model_separate_from_routed_traffic() -> Result<(.get(model).ok_or_else(|| SwitchyardError::Other("backend row missing".to_string()))?;assert_eq!(backend.prompt_tokens, 1_000_000);+ assert_eq!(backend.max_observed_context_tokens, 1_000_000);assert_eq!(backend.calls, 1);// Classifier bucket: same model id, separate row, separate counts.@@ -662,6 +690,7 @@ fn classifier_bucket_keeps_same_model_separate_from_routed_traffic() -> Result<(.get(model).ok_or_else(|| SwitchyardError::Other("classifier row missing".to_string()))?;assert_eq!(classifier.prompt_tokens, 500_000);+ assert_eq!(classifier.max_observed_context_tokens, 500_000);assert_eq!(classifier.calls, 1);assert_eq!(snapshot.classifier.total_requests, 1);@@ -673,16 +702,51 @@ fn classifier_bucket_keeps_same_model_separate_from_routed_traffic() -> Result<(Ok(())}+#[test]+fn planner_bucket_tracks_max_observed_context_tokens() -> Result<()> {+ let accumulator = StatsAccumulator::new();+ for (prompt_tokens, completion_tokens) in [(120, 0), (300, 5), (200, 200)] {+ accumulator.record_planner_usage(+ "planner/model",+ TokenUsage {+ prompt_tokens,+ completion_tokens,+ ..TokenUsage::default()+ },+ None,+ )?;+ }++ let snapshot = accumulator.snapshot()?;+ let planner = snapshot+ .planner+ .models+ .get("planner/model")+ .ok_or_else(|| SwitchyardError::Other("planner row missing".to_string()))?;+ assert_eq!(planner.prompt_tokens, 620);+ assert_eq!(planner.max_observed_context_tokens, 400);+ Ok(())+}+#[test]fn classifier_latency_lands_on_classifier_model_call_latency() -> Result<()> {let accumulator = StatsAccumulator::new();- accumulator.record_classifier_usage("claude-sonnet-4-6", TokenUsage::default(), Some(15.0))?;+ accumulator.record_classifier_usage(+ "claude-sonnet-4-6",+ TokenUsage {+ prompt_tokens: 10,+ completion_tokens: 5,+ ..TokenUsage::default()+ },+ Some(15.0),+ )?;let snapshot = accumulator.snapshot()?;let row = snapshot.classifier.models.get("claude-sonnet-4-6").ok_or_else(|| SwitchyardError::Other("classifier row missing".to_string()))?;+ assert_eq!(row.max_observed_context_tokens, 15);assert_eq!(row.model_call_latency.count, 1);assert_eq!(row.model_call_latency.total_ms, 15.0);Ok(())diff --git a/switchyard/lib/live_stats_collector.py b/switchyard/lib/live_stats_collector.pyindex 5bb4ec543..afb865087 100644--- a/switchyard/lib/live_stats_collector.py+++ b/switchyard/lib/live_stats_collector.py@@ -93,6 +93,8 @@ class ModelStats:sum ``input_tokens + cache_creation_input_tokens +cache_read_input_tokens`` (Anthropic reports them as siblings, notparent/child). For OpenAI it is ``usage.prompt_tokens``.+ * ``max_observed_context_tokens`` — largest prompt-plus-completion+ token count observed in one completed response.* ``completion_tokens`` — total output tokens.* ``total_tokens`` — ``prompt_tokens + completion_tokens``.* ``reasoning_tokens`` — subset of ``completion_tokens`` for chain-of-@@ -108,6 +110,7 @@ class ModelStats:tier: str = ""calls: int = 0prompt_tokens: int = 0+ max_observed_context_tokens: int = 0completion_tokens: int = 0total_tokens: int = 0reasoning_tokens: int = 0@@ -157,6 +160,10 @@ class LiveStatsCollector:self._models[model] = bucketbucket.calls += 1bucket.prompt_tokens += prompt_tokens+ bucket.max_observed_context_tokens = max(+ bucket.max_observed_context_tokens,+ prompt_tokens + completion_tokens,+ )bucket.completion_tokens += completion_tokensbucket.total_tokens += prompt_tokens + completion_tokensbucket.reasoning_tokens += reasoning_tokens@@ -214,6 +221,7 @@ class LiveStatsCollector:"tier": str,"calls": int,"prompt_tokens": int,+ "max_observed_context_tokens": int,"completion_tokens": int,"total_tokens": int,"reasoning_tokens": int,@@ -243,6 +251,7 @@ class LiveStatsCollector:"tier": b.tier,"calls": b.calls,"prompt_tokens": b.prompt_tokens,+ "max_observed_context_tokens": b.max_observed_context_tokens,"completion_tokens": b.completion_tokens,"total_tokens": b.total_tokens,"reasoning_tokens": b.reasoning_tokens,diff --git a/tests/test_live_stats_collector.py b/tests/test_live_stats_collector.pyindex ef7550ef0..e8de312ff 100644--- a/tests/test_live_stats_collector.py+++ b/tests/test_live_stats_collector.py@@ -54,6 +54,7 @@ def test_record_accumulates_across_calls():assert s.request_count == 2assert s.prompt_tokens == 30assert s.completion_tokens == 15+ assert c.to_dict()["models"]["unknown"]["max_observed_context_tokens"] == 30def test_snapshot_returns_copy():diff --git a/tests/test_metrics_endpoint.py b/tests/test_metrics_endpoint.pyindex af98252e4..e3dbd7678 100644--- a/tests/test_metrics_endpoint.py+++ b/tests/test_metrics_endpoint.py@@ -53,6 +53,7 @@ async def test_v1_stats_returns_existing_json_shape(assert body["total_requests"] == 1assert body["models"]["m"]["calls"] == 1assert body["models"]["m"]["prompt_tokens"] == 5+ assert body["models"]["m"]["max_observed_context_tokens"] == 8assert "cost_estimate" in body