Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(metrics): extend LLM latency histogram buckets (#385)
Yahoo1bbb8067eb4d
diff --git a/crates/switchyard-server/src/metrics.rs b/crates/switchyard-server/src/metrics.rsindex 2469e0251..b6ba04f5c 100644--- a/crates/switchyard-server/src/metrics.rs+++ b/crates/switchyard-server/src/metrics.rs@@ -19,6 +19,13 @@ const ROUTING_OVERHEAD_BUCKETS_MS: &[f64] = &[0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0, 2500.0, 5000.0,];+/// Bucket boundaries for model-call and end-to-end LLM latency histograms.+/// Retains the SDK defaults through 10 seconds and extends them for long generations.+const LLM_LATENCY_BUCKETS_MS: &[f64] = &[+ 0.0, 5.0, 10.0, 25.0, 50.0, 75.0, 100.0, 250.0, 500.0, 750.0, 1000.0, 2500.0, 5000.0, 7500.0,+ 10_000.0, 15_000.0, 30_000.0, 60_000.0, 120_000.0, 300_000.0,+];+struct Metrics {registry: Registry,provider: SdkMeterProvider,@@ -43,6 +50,7 @@ fn initialize() -> Result<Metrics, String> {let mut builder = SdkMeterProvider::builder().with_reader(exporter).with_view(routing_overhead_buckets)+ .with_view(llm_latency_buckets).with_resource(crate::observability::resource());if crate::observability::otlp_enabled("METRICS") {let exporter = opentelemetry_otlp::MetricExporter::builder()@@ -84,6 +92,22 @@ fn routing_overhead_buckets(instrument: &Instrument) -> Option<Stream> {.ok()}+fn llm_latency_buckets(instrument: &Instrument) -> Option<Stream> {+ if !matches!(+ instrument.name(),+ "switchyard.model_call_latency_ms" | "switchyard.total_latency_ms"+ ) {+ return None;+ }+ Stream::builder()+ .with_aggregation(Aggregation::ExplicitBucketHistogram {+ boundaries: LLM_LATENCY_BUCKETS_MS.to_vec(),+ record_min_max: true,+ })+ .build()+ .ok()+}+/// Make the metrics exist before they get a hit. Nicer for dashboards but not really necessary./// The HTTP status codes we seed are somewhat arbitrary.fn seed_outcome_metrics() {diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rsindex 456ed9192..a0e38159b 100644--- a/crates/switchyard-server/tests/server.rs+++ b/crates/switchyard-server/tests/server.rs@@ -603,6 +603,15 @@ async fn metrics_exposes_switchyard_otel_instruments() -> TestResult {).is_some());+ for metric in [+ "switchyard_model_call_latency_ms_bucket",+ "switchyard_total_latency_ms_bucket",+ ] {+ assert!(+ metric_line(metrics, metric, &[("model", MODEL), ("le", "300000")]).is_some(),+ "missing five-minute bucket for {metric}"+ );+ }assert!(metric_line(metrics,