Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(stats): track max observed context tokens per model (#24)

Aloys Bailletbacab4cf617a
diff --git a/crates/switchyard-components/src/stats/accumulator.rs b/crates/switchyard-components/src/stats/accumulator.rs
index ee2f72602..61a99f835 100644
--- a/crates/switchyard-components/src/stats/accumulator.rs
+++ b/crates/switchyard-components/src/stats/accumulator.rs
@@ -173,6 +173,9 @@ impl StatsAccumulator {
{
let stats = inner.model_stats_mut(model.clone());
stats.prompt_tokens = stats.prompt_tokens.saturating_add(usage.prompt_tokens);
+ stats.max_observed_context_tokens = stats
+ .max_observed_context_tokens
+ .max(usage.prompt_tokens.saturating_add(usage.completion_tokens));
stats.completion_tokens = stats
.completion_tokens
.saturating_add(usage.completion_tokens);
@@ -242,6 +245,9 @@ impl StatsAccumulator {
let stats = inner.classifier_stats_mut(model.into());
stats.calls = stats.calls.saturating_add(1);
stats.prompt_tokens = stats.prompt_tokens.saturating_add(usage.prompt_tokens);
+ stats.max_observed_context_tokens = stats
+ .max_observed_context_tokens
+ .max(usage.prompt_tokens.saturating_add(usage.completion_tokens));
stats.completion_tokens = stats
.completion_tokens
.saturating_add(usage.completion_tokens);
@@ -316,6 +322,9 @@ impl StatsAccumulator {
let stats = inner.planner_stats_mut(model.into());
stats.calls = stats.calls.saturating_add(1);
stats.prompt_tokens = stats.prompt_tokens.saturating_add(usage.prompt_tokens);
+ stats.max_observed_context_tokens = stats
+ .max_observed_context_tokens
+ .max(usage.prompt_tokens.saturating_add(usage.completion_tokens));
stats.completion_tokens = stats
.completion_tokens
.saturating_add(usage.completion_tokens);
@@ -554,6 +563,7 @@ fn build_model_snapshots(
errors: stats.errors,
request_pct,
prompt_tokens: stats.prompt_tokens,
+ max_observed_context_tokens: stats.max_observed_context_tokens,
completion_tokens: stats.completion_tokens,
total_tokens: token_total,
token_pct,
@@ -610,6 +620,7 @@ struct ModelStats {
calls: u64,
errors: u64,
prompt_tokens: u64,
+ max_observed_context_tokens: u64,
completion_tokens: u64,
cached_tokens: u64,
cache_creation_tokens: u64,
@@ -796,6 +807,8 @@ pub struct ModelStatsSnapshot {
pub errors: u64,
pub request_pct: f64,
pub prompt_tokens: u64,
+ /// Largest prompt-plus-completion token count observed in one completed response.
+ pub max_observed_context_tokens: u64,
pub completion_tokens: u64,
pub total_tokens: u64,
pub token_pct: f64,
diff --git a/crates/switchyard-components/tests/stats_accumulator.rs b/crates/switchyard-components/tests/stats_accumulator.rs
index 9b005386d..6ba105904 100644
--- a/crates/switchyard-components/tests/stats_accumulator.rs
+++ b/crates/switchyard-components/tests/stats_accumulator.rs
@@ -87,6 +87,7 @@ fn accumulator_matches_python_two_tier_snapshot_contract() -> Result<()> {
assert_eq!(strong.tier.as_deref(), Some("strong"));
assert_eq!(strong.request_pct, 50.0);
assert_eq!(strong.token_pct, 73.53);
+ assert_eq!(strong.max_observed_context_tokens, 125);
assert_eq!(strong.avg_prompt_tokens, 100.0);
assert_eq!(strong.avg_completion_tokens, 25.0);
assert_eq!(strong.cache_hit_rate, 0.1);
@@ -110,6 +111,32 @@ fn accumulator_matches_python_two_tier_snapshot_contract() -> Result<()> {
Ok(())
}
+#[test]
+fn accumulator_tracks_max_observed_context_tokens_per_model() -> Result<()> {
+ let accumulator = StatsAccumulator::new();
+ for (prompt_tokens, completion_tokens) in [(100, 10), (90, 50), (120, 5)] {
+ accumulator.record_success("model-a", None, None)?;
+ accumulator.record_usage(
+ "model-a",
+ TokenUsage {
+ prompt_tokens,
+ completion_tokens,
+ ..TokenUsage::default()
+ },
+ None,
+ None,
+ None,
+ )?;
+ }
+
+ let snapshot = accumulator.snapshot()?;
+ let model = model_stats(&snapshot, "model-a")?;
+ assert_eq!(model.prompt_tokens, 310);
+ assert_eq!(model.completion_tokens, 65);
+ assert_eq!(model.max_observed_context_tokens, 140);
+ Ok(())
+}
+
// Switch-aware theoretical: cold on switch-in, partial credit on switching back.
#[test]
fn theoretical_is_switch_aware_across_a_model_switch_and_back() -> Result<()> {
@@ -653,6 +680,7 @@ fn classifier_bucket_keeps_same_model_separate_from_routed_traffic() -> Result<(
.get(model)
.ok_or_else(|| SwitchyardError::Other("backend row missing".to_string()))?;
assert_eq!(backend.prompt_tokens, 1_000_000);
+ assert_eq!(backend.max_observed_context_tokens, 1_000_000);
assert_eq!(backend.calls, 1);
// Classifier bucket: same model id, separate row, separate counts.
@@ -662,6 +690,7 @@ fn classifier_bucket_keeps_same_model_separate_from_routed_traffic() -> Result<(
.get(model)
.ok_or_else(|| SwitchyardError::Other("classifier row missing".to_string()))?;
assert_eq!(classifier.prompt_tokens, 500_000);
+ assert_eq!(classifier.max_observed_context_tokens, 500_000);
assert_eq!(classifier.calls, 1);
assert_eq!(snapshot.classifier.total_requests, 1);
@@ -673,16 +702,51 @@ fn classifier_bucket_keeps_same_model_separate_from_routed_traffic() -> Result<(
Ok(())
}
+#[test]
+fn planner_bucket_tracks_max_observed_context_tokens() -> Result<()> {
+ let accumulator = StatsAccumulator::new();
+ for (prompt_tokens, completion_tokens) in [(120, 0), (300, 5), (200, 200)] {
+ accumulator.record_planner_usage(
+ "planner/model",
+ TokenUsage {
+ prompt_tokens,
+ completion_tokens,
+ ..TokenUsage::default()
+ },
+ None,
+ )?;
+ }
+
+ let snapshot = accumulator.snapshot()?;
+ let planner = snapshot
+ .planner
+ .models
+ .get("planner/model")
+ .ok_or_else(|| SwitchyardError::Other("planner row missing".to_string()))?;
+ assert_eq!(planner.prompt_tokens, 620);
+ assert_eq!(planner.max_observed_context_tokens, 400);
+ Ok(())
+}
+
#[test]
fn classifier_latency_lands_on_classifier_model_call_latency() -> Result<()> {
let accumulator = StatsAccumulator::new();
- accumulator.record_classifier_usage("claude-sonnet-4-6", TokenUsage::default(), Some(15.0))?;
+ accumulator.record_classifier_usage(
+ "claude-sonnet-4-6",
+ TokenUsage {
+ prompt_tokens: 10,
+ completion_tokens: 5,
+ ..TokenUsage::default()
+ },
+ Some(15.0),
+ )?;
let snapshot = accumulator.snapshot()?;
let row = snapshot
.classifier
.models
.get("claude-sonnet-4-6")
.ok_or_else(|| SwitchyardError::Other("classifier row missing".to_string()))?;
+ assert_eq!(row.max_observed_context_tokens, 15);
assert_eq!(row.model_call_latency.count, 1);
assert_eq!(row.model_call_latency.total_ms, 15.0);
Ok(())
diff --git a/switchyard/lib/live_stats_collector.py b/switchyard/lib/live_stats_collector.py
index 5bb4ec543..afb865087 100644
--- a/switchyard/lib/live_stats_collector.py
+++ b/switchyard/lib/live_stats_collector.py
@@ -93,6 +93,8 @@ class ModelStats:
sum ``input_tokens + cache_creation_input_tokens +
cache_read_input_tokens`` (Anthropic reports them as siblings, not
parent/child). For OpenAI it is ``usage.prompt_tokens``.
+ * ``max_observed_context_tokens`` — largest prompt-plus-completion
+ token count observed in one completed response.
* ``completion_tokens`` — total output tokens.
* ``total_tokens`` — ``prompt_tokens + completion_tokens``.
* ``reasoning_tokens`` — subset of ``completion_tokens`` for chain-of-
@@ -108,6 +110,7 @@ class ModelStats:
tier: str = ""
calls: int = 0
prompt_tokens: int = 0
+ max_observed_context_tokens: int = 0
completion_tokens: int = 0
total_tokens: int = 0
reasoning_tokens: int = 0
@@ -157,6 +160,10 @@ class LiveStatsCollector:
self._models[model] = bucket
bucket.calls += 1
bucket.prompt_tokens += prompt_tokens
+ bucket.max_observed_context_tokens = max(
+ bucket.max_observed_context_tokens,
+ prompt_tokens + completion_tokens,
+ )
bucket.completion_tokens += completion_tokens
bucket.total_tokens += prompt_tokens + completion_tokens
bucket.reasoning_tokens += reasoning_tokens
@@ -214,6 +221,7 @@ class LiveStatsCollector:
"tier": str,
"calls": int,
"prompt_tokens": int,
+ "max_observed_context_tokens": int,
"completion_tokens": int,
"total_tokens": int,
"reasoning_tokens": int,
@@ -243,6 +251,7 @@ class LiveStatsCollector:
"tier": b.tier,
"calls": b.calls,
"prompt_tokens": b.prompt_tokens,
+ "max_observed_context_tokens": b.max_observed_context_tokens,
"completion_tokens": b.completion_tokens,
"total_tokens": b.total_tokens,
"reasoning_tokens": b.reasoning_tokens,
diff --git a/tests/test_live_stats_collector.py b/tests/test_live_stats_collector.py
index ef7550ef0..e8de312ff 100644
--- a/tests/test_live_stats_collector.py
+++ b/tests/test_live_stats_collector.py
@@ -54,6 +54,7 @@ def test_record_accumulates_across_calls():
assert s.request_count == 2
assert s.prompt_tokens == 30
assert s.completion_tokens == 15
+ assert c.to_dict()["models"]["unknown"]["max_observed_context_tokens"] == 30
def test_snapshot_returns_copy():
diff --git a/tests/test_metrics_endpoint.py b/tests/test_metrics_endpoint.py
index af98252e4..e3dbd7678 100644
--- a/tests/test_metrics_endpoint.py
+++ b/tests/test_metrics_endpoint.py
@@ -53,6 +53,7 @@ async def test_v1_stats_returns_existing_json_shape(
assert body["total_requests"] == 1
assert body["models"]["m"]["calls"] == 1
assert body["models"]["m"]["prompt_tokens"] == 5
+ assert body["models"]["m"]["max_observed_context_tokens"] == 8
assert "cost_estimate" in body