Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(costs): save call costs and share pricing (#1045)
Elyas Mehtabuddin7dddc65a0997
diff --git a/Cargo.lock b/Cargo.lockindex 705b86d08..19bf434ba 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -3640,6 +3640,17 @@ dependencies = ["tokio",]+[[package]]+name = "switchyard-costs"+version = "0.3.0"+dependencies = [+ "chrono",+ "serde",+ "serde_json",+ "switchyard-protocol",+ "toml 0.9.12+spec-1.1.0",+]+[[package]]name = "switchyard-desktop"version = "0.3.0"@@ -3723,6 +3734,8 @@ dependencies = ["objc2-foundation","serde","serde_json",+ "switchyard-costs",+ "switchyard-protocol","tempfile","toml 0.9.12+spec-1.1.0","tray-icon",@@ -3859,6 +3872,7 @@ dependencies = ["rustls","serde","serde_json",+ "switchyard-costs","switchyard-libsy","switchyard-llm-client","switchyard-protocol",diff --git a/Cargo.toml b/Cargo.tomlindex e13c41b0d..60228cabd 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -16,6 +16,7 @@ members = ["crates/switchyard-runner","crates/switchyard-desktop","crates/switchyard-menubar",+ "crates/switchyard-costs","crates/switchyard-server","crates/switchyard-skill-distillation","crates/switchyard-soak",@@ -49,6 +50,7 @@ regex = "1"reqwest = { version = "0.13.4", default-features = false, features = ["json", "rustls", "stream"] }serde = { version = "1", features = ["derive"] }serde_json = { version = "1", features = ["preserve_order"] }+switchyard-costs = { path = "crates/switchyard-costs", version = "0.3.0" }switchyard-libsy = { path = "crates/libsy", version = "0.3.0" }switchyard-llm-client = { path = "crates/libsy-llm-client", version = "0.3.0" }switchyard-protocol = { path = "crates/protocol", version = "0.3.0" }diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rsindex 4c3d82a82..3e4fc675a 100644--- a/crates/libsy-llm-client/src/client.rs+++ b/crates/libsy-llm-client/src/client.rs@@ -660,7 +660,7 @@ impl TranslatingLlmClient {})?;// Map a provider's failed generation to 502, even under HTTP 200.// Redact forwarded credentials before returning the provider error.- let agg =+ let mut agg =observed_translation::decode_response(&body, wire_format).map_err(|error| {match error {TranslationError::UpstreamFailure { error } => {@@ -676,6 +676,14 @@ impl TranslatingLlmClient {error => LlmClientError::ResponseTranslation(error.to_string()),}})?;+ // The gateway may report a buffered charge only in its response header.+ if agg.usage.provider_cost.is_none() {+ agg.usage.provider_cost = upstream_headers+ .get("x-litellm-response-cost")+ .and_then(|value| value.to_str().ok())+ .and_then(|value| value.trim().parse::<f64>().ok())+ .filter(|usd| usd.is_finite() && *usd >= 0.0);+ }(LlmResponse::Agg(agg), upstream_headers)}};diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rsindex 00bdf76c8..661e05c6b 100644--- a/crates/libsy-llm-client/src/run.rs+++ b/crates/libsy-llm-client/src/run.rs@@ -33,7 +33,7 @@ use switchyard_libsy::{use switchyard_protocol::{AggLlmResponse, LlmClientError, LlmResponse, LlmResponseChunk, LlmResponseStream, Message,ModelId, Request, Response, ResponseAccumulator, RoutedDecisionClient, RoutedLlmClient,- RoutingFallbackReason, WireFormat,+ RoutingFallbackReason, Usage, WireFormat,};use switchyard_translation::prepare_request_for_target;@@ -376,10 +376,13 @@ async fn call_one(// the provider's, so it belongs in the routing overhead.let client = clients.route(model_id);let started = Instant::now();+ let mut routing_usage = None;let result = async {let mut response = client?.call(request).await?;if is_routing && let LlmResponse::Stream(chunks) = response.llm_response {- response.llm_response = LlmResponse::Stream(buffer_routing_stream(chunks).await?);+ let (chunks, usage) = buffer_routing_stream(chunks).await?;+ routing_usage = usage;+ response.llm_response = LlmResponse::Stream(chunks);}Ok(response)}@@ -405,19 +408,22 @@ async fn call_one(.as_ref().ok().and_then(|response| response.llm_response.as_agg())- .map(|response| response.usage.clone()),+ .map(|response| response.usage.clone())+ .or(routing_usage),});result}-// Preserve signed provider events while checking the complete routing response.+// Routing logs need the final usage, but replay must keep signed provider events intact.async fn buffer_routing_stream(mut chunks: LlmResponseStream,-) -> std::result::Result<LlmResponseStream, LlmClientError> {+) -> std::result::Result<(LlmResponseStream, Option<Usage>), LlmClientError> {let mut events = Vec::new();+ let mut usage = None;while let Some(event) = chunks.try_next().await? {for chunk in event.normalized() {match chunk {+ LlmResponseChunk::Usage(latest) => usage = Some(latest.clone()),LlmResponseChunk::DecodeError { message } => {return Err(LlmClientError::ResponseTranslation(message.clone()));}@@ -432,7 +438,7 @@ async fn buffer_routing_stream(}events.push(event);}- Ok(stream::iter(events.into_iter().map(Ok)).boxed())+ Ok((stream::iter(events.into_iter().map(Ok)).boxed(), usage))}/// Whether a failed candidate is worth routing around.diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rsindex 27a5f3862..ec16a2c0f 100644--- a/crates/libsy-llm-client/tests/observability.rs+++ b/crates/libsy-llm-client/tests/observability.rs@@ -996,6 +996,7 @@ async fn successful_run_records_metrics_spans_and_outcome_metadata() -> switchyatotal_tokens: Some(25),reasoning_tokens: Some(2),cache: Usage::cache_details(Some(3), Some(4)),+ provider_cost: None,},}) as Arc<dyn RoutedLlmClient>;let mut request = request_with_metadata("obs-session-1", "obs-corr-1");diff --git a/crates/protocol/src/llm.rs b/crates/protocol/src/llm.rsindex 9429354e5..cb3731aef 100644--- a/crates/protocol/src/llm.rs+++ b/crates/protocol/src/llm.rs@@ -559,13 +559,16 @@ pub struct LlmRequest {/// Normalized token usage counts.#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]pub struct Usage {+ /// The upstream provider reports this monetary charge in USD.+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub provider_cost: Option<f64>,/// Non-cached input tokens. Provider codecs normalize aggregate OpenAI/// input counts by subtracting the cache detail fields.pub input_tokens: Option<u64>,/// Cache-read and cache-creation token detail, when reported.#[serde(flatten)]pub cache: Option<Box<InputCacheUsage>>,- /// Generated output tokens, excluding reasoning detail when the provider reports it separately.+ /// The output count includes reasoning tokens when the provider counts them.pub output_tokens: Option<u64>,/// Provider-reported or codec-computed total token count.///@@ -579,6 +582,9 @@ pub struct Usage {/// Optional cache-token detail kept out of the common, cache-free usage allocation.#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]pub struct InputCacheUsage {+ /// The provider reports these cache-write tokens at the one-hour rate.+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub cache_creation_1h_input_tokens: Option<u64>,/// Input tokens read from a provider cache.pub cached_input_tokens: Option<u64>,/// Input tokens written into a provider cache.@@ -595,6 +601,7 @@ impl Usage {return None;}Some(Box::new(InputCacheUsage {+ cache_creation_1h_input_tokens: None,cached_input_tokens,cache_creation_input_tokens,}))diff --git a/crates/switchyard-costs/Cargo.toml b/crates/switchyard-costs/Cargo.tomlnew file mode 100644index 000000000..7176256d8--- /dev/null+++ b/crates/switchyard-costs/Cargo.toml@@ -0,0 +1,19 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++[package]+name = "switchyard-costs"+version.workspace = true+description = "Shared token pricing and recorded cost totals for Switchyard"+authors.workspace = true+edition.workspace = true+license.workspace = true+repository.workspace = true+rust-version.workspace = true++[dependencies]+chrono = { version = "0.4", default-features = false, features = ["std"] }+switchyard-protocol.workspace = true+serde.workspace = true+serde_json.workspace = true+toml = "0.9"diff --git a/crates/switchyard-costs/src/anthropic.rs b/crates/switchyard-costs/src/anthropic.rsnew file mode 100644index 000000000..99b760142--- /dev/null+++ b/crates/switchyard-costs/src/anthropic.rs@@ -0,0 +1,27 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Anthropic prices cache writes at separate rates for five-minute and one-hour lifetimes.++use switchyard_protocol::Usage;++/// The caller divides this weighted token sum by one million to obtain USD.+pub(super) fn cache_write_cost(+ usage: &Usage,+ short_rate: f64,+ long_rate: Option<f64>,+) -> Option<f64> {+ let written = usage.cache_creation_input_tokens().unwrap_or(0);+ let one_hour = usage+ .cache+ .as_ref()+ .and_then(|cache| cache.cache_creation_1h_input_tokens)+ .unwrap_or(0);+ let five_minute = written.checked_sub(one_hour)?;+ let long_cost = if one_hour > 0 {+ one_hour as f64 * long_rate?+ } else {+ 0.0+ };+ Some(five_minute as f64 * short_rate + long_cost)+}diff --git a/crates/switchyard-costs/src/lib.rs b/crates/switchyard-costs/src/lib.rsnew file mode 100644index 000000000..a0001ea19--- /dev/null+++ b/crates/switchyard-costs/src/lib.rs@@ -0,0 +1,198 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! This module prices observed calls and keeps missing usage or rates unknown.++mod anthropic;++use std::collections::HashMap;+use std::path::Path;++use serde::{Deserialize, Serialize};+use switchyard_protocol::Usage;++/// Pricing maps served model IDs to token rates in USD per million tokens.+#[derive(Clone, Debug, Default)]+pub struct Pricing(HashMap<String, Rates>);++impl<'de> Deserialize<'de> for Pricing {+ fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {+ // This visitor rejects duplicate IDs before a map can overwrite a rate table.+ struct UniqueModels;+ impl<'de> serde::de::Visitor<'de> for UniqueModels {+ type Value = Pricing;+ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ formatter.write_str("a map of unique model IDs to token rates")+ }+ fn visit_map<M: serde::de::MapAccess<'de>>(+ self,+ mut map: M,+ ) -> Result<Pricing, M::Error> {+ let mut rates = HashMap::new();+ while let Some((model, rate)) = map.next_entry::<String, Rates>()? {+ if rates.insert(model.clone(), rate).is_some() {+ return Err(serde::de::Error::custom(format!(+ "duplicate pricing model {model:?}"+ )));+ }+ }+ Ok(Pricing(rates))+ }+ }+ deserializer.deserialize_map(UniqueModels)+ }+}++#[derive(Clone, Debug, Deserialize)]+#[serde(deny_unknown_fields)]+struct Rates {+ input: f64,+ output: f64,+ cache_read: f64,+ cache_write: f64,+ cache_write_1h: Option<f64>,+}++/// This value identifies how a saved monetary amount was obtained.+#[derive(Clone, Copy, Debug, PartialEq, Deserialize, Serialize)]+#[serde(rename_all = "snake_case")]+pub enum CostSource {+ /// The provider supplied this charge.+ ProviderReported,+ /// Configured token rates supplied this estimate.+ Estimated,+}++/// This record stores a charge or estimate for one call.+#[derive(Clone, Copy, Debug, PartialEq, Deserialize, Serialize)]+pub struct RecordedCost {+ /// The amount is in USD.+ pub usd: f64,+ /// The source distinguishes charges from estimates.+ pub source: CostSource,+}++impl Pricing {+ /// This method loads and validates a JSON pricing file.+ pub fn load(path: &Path) -> Result<Self, String> {+ let bytes = std::fs::read(path)+ .map_err(|error| format!("read pricing file {}: {error}", path.display()))?;+ Self::parse(&bytes)+ .map_err(|error| format!("invalid pricing file {}: {error}", path.display()))+ }++ /// This function reads a pricing file's JSON and checks every model ID and rate.+ pub fn parse(json: &[u8]) -> Result<Self, String> {+ let pricing: Self = serde_json::from_slice(json).map_err(|error| error.to_string())?;+ for (model, rate) in &pricing.0 {+ if model.trim().is_empty()+ || model.trim() != model+ || [+ Some(rate.input),+ Some(rate.output),+ Some(rate.cache_read),+ Some(rate.cache_write),+ rate.cache_write_1h,+ ]+ .into_iter()+ .flatten()+ .any(|value| !value.is_finite() || !(0.0..=10000.0).contains(&value))+ {+ return Err(format!(+ "invalid pricing for model {model:?}: use a nonempty model ID and rates from 0 to 10000 USD per million tokens"+ ));+ }+ }+ Ok(pricing)+ }++ /// Provider charges take precedence because token rates may omit provider surcharges.+ pub fn cost(&self, model: &str, usage: &Usage) -> Option<RecordedCost> {+ if let Some(usd) = usage+ .provider_cost+ .filter(|value| value.is_finite() && *value >= 0.0)+ {+ return Some(RecordedCost {+ usd,+ source: CostSource::ProviderReported,+ });+ }+ self.estimate(model, usage)+ }++ /// This method uses token rates alone so baseline estimates exclude provider charges.+ pub fn estimate(&self, model: &str, usage: &Usage) -> Option<RecordedCost> {+ let rate = self.0.get(model)?;+ let input = usage.input_tokens?;+ let output = usage.output_tokens?;+ let read = usage.cached_input_tokens().unwrap_or(0);+ let usd = (input as f64 * rate.input+ + output as f64 * rate.output+ + read as f64 * rate.cache_read+ + anthropic::cache_write_cost(usage, rate.cache_write, rate.cache_write_1h)?)+ / 1_000_000.0;+ Some(RecordedCost {+ usd,+ source: CostSource::Estimated,+ })+ }+}++/// The known subtotal remains visible when any call has unknown cost.+#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]+pub struct CostTotals {+ /// This subtotal includes every known cost.+ pub known_usd: f64,+ /// This count includes calls whose cost is unknown.+ pub unknown_calls: u64,+ /// This count includes calls priced from configured rates.+ pub estimated_calls: u64,+ /// This count includes calls with a provider charge.+ pub provider_reported_calls: u64,+ /// The total is absent when any call has unknown cost.+ pub total_usd: Option<f64>,+}++impl Default for CostTotals {+ fn default() -> Self {+ Self {+ known_usd: 0.0,+ unknown_calls: 0,+ estimated_calls: 0,+ provider_reported_calls: 0,+ total_usd: Some(0.0),+ }+ }+}++impl CostTotals {+ /// This method adds one call and keeps the total unknown if any cost is unknown.+ pub fn add(&mut self, cost: Option<RecordedCost>) {+ match cost.filter(|cost| {+ cost.usd.is_finite() && cost.usd >= 0.0 && (self.known_usd + cost.usd).is_finite()+ }) {+ Some(cost) => {+ self.known_usd += cost.usd;+ match cost.source {+ CostSource::Estimated => self.estimated_calls += 1,+ CostSource::ProviderReported => self.provider_reported_calls += 1,+ }+ }+ None => self.unknown_calls += 1,+ }+ self.total_usd = (self.unknown_calls == 0).then_some(self.known_usd);+ }++ /// This method combines disjoint groups of calls without repricing them.+ pub fn merge(&mut self, other: &Self) {+ if (self.known_usd + other.known_usd).is_finite() {+ self.known_usd += other.known_usd;+ } else {+ self.unknown_calls += 1;+ }+ self.unknown_calls += other.unknown_calls;+ self.estimated_calls += other.estimated_calls;+ self.provider_reported_calls += other.provider_reported_calls;+ self.total_usd = (self.unknown_calls == 0).then_some(self.known_usd);+ }+}diff --git a/crates/switchyard-costs/src/main.rs b/crates/switchyard-costs/src/main.rsnew file mode 100644index 000000000..dc3f7b7f4--- /dev/null+++ b/crates/switchyard-costs/src/main.rs@@ -0,0 +1,172 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! This command migrates old settings and logs without changing the originals.++use std::io::{BufRead, Write};+use std::path::Path;++use serde::Deserialize;+use serde_json::{Value, json};+use switchyard_costs::{Pricing, RecordedCost};+use switchyard_protocol::Usage;++#[derive(Deserialize)]+#[serde(deny_unknown_fields)]+struct OldRate {+ input_per_mtok: f64,+ output_per_mtok: f64,+ cached_input_per_mtok: Option<f64>,+}++#[derive(Deserialize)]+struct Record {+ ts: String,+ #[serde(default, rename = "tier")]+ _tier: String,+ model: String,+ prompt_tokens: Option<u64>,+ completion_tokens: Option<u64>,+ cached_tokens: Option<u64>,+ cache_creation_tokens: Option<u64>,+ cache_creation_1h_tokens: Option<u64>,+ // The nested option distinguishes an absent cost from an explicit unknown cost.+ #[serde(default, deserialize_with = "present_cost")]+ cost: Option<Option<RecordedCost>>,+}++// An explicit null remains unknown instead of being reconstructed from token rates.+fn present_cost<'de, D: serde::Deserializer<'de>>(+ deserializer: D,+) -> Result<Option<Option<RecordedCost>>, D::Error> {+ Option::deserialize(deserializer).map(Some)+}++// Only records without a cost field receive reconstructed estimates.+fn migrate(+ settings: &Path,+ log: &Path,+ destination: &Path,+) -> Result<(), Box<dyn std::error::Error>> {+ let mut config: toml::Table = std::fs::read_to_string(settings)?.parse()?;+ let rates = match config.remove("prices") {+ Some(old) => {+ if config.contains_key("pricing_file") {+ return Err("settings contain both prices and pricing_file".into());+ }+ let old: std::collections::BTreeMap<String, OldRate> = old.try_into()?;+ Value::Object(old.into_iter().map(|(model, rate)| (model, json!({+ "input":rate.input_per_mtok, "output":rate.output_per_mtok,+ "cache_read":rate.cached_input_per_mtok.unwrap_or(rate.input_per_mtok),+ "cache_write":rate.input_per_mtok,+ }))).collect())+ }+ None => return Err("migration requires the old settings with a [prices] table".into()),+ };+ let rates = serde_json::to_vec_pretty(&rates)?;+ let pricing = Pricing::parse(&rates)?;+ let destination = std::path::absolute(destination)?;+ config.insert(+ "pricing_file".into(),+ toml::Value::String(+ destination+ .join("prices.json")+ .to_string_lossy()+ .into_owned(),+ ),+ );+ config.insert(+ "routing_log".into(),+ toml::Value::String(+ destination+ .join("routing.jsonl")+ .to_string_lossy()+ .into_owned(),+ ),+ );+ // Creating a fresh directory prevents overwriting inputs or earlier migration results.+ std::fs::create_dir(&destination)?;+ let result = (|| -> Result<(), Box<dyn std::error::Error>> {+ let mut output =+ std::io::BufWriter::new(std::fs::File::create(destination.join("routing.jsonl"))?);+ let mut source = std::io::BufReader::new(std::fs::File::open(log)?);+ let mut line = String::new();+ loop {+ line.clear();+ if source.read_line(&mut line)? == 0 {+ break;+ }+ if !line.ends_with('\n') {+ return Err("unfinished log record; stop the writer before migrating".into());+ }+ // Typed parsing catches duplicate known fields before Value could overwrite them.+ let record: Record = serde_json::from_str(&line)?;+ chrono::DateTime::parse_from_rfc3339(&record.ts)?;+ if record.model.trim().is_empty() {+ return Err("log record has an empty model".into());+ }+ if let Some(Some(cost)) = record.cost+ && (!cost.usd.is_finite() || cost.usd < 0.0)+ {+ return Err("log record has an invalid cost".into());+ }+ let mut value: Value = serde_json::from_str(&line)?;+ if !value.is_object() {+ return Err("log record must be a JSON object".into());+ }+ if record.cost.is_none() {+ let read = record.cached_tokens.unwrap_or(0);+ let written = record.cache_creation_tokens.unwrap_or(0);+ let mut cache = Usage::cache_details(Some(read), Some(written));+ if let Some(tokens) = record.cache_creation_1h_tokens {+ cache+ .get_or_insert_with(Default::default)+ .cache_creation_1h_input_tokens = Some(tokens);+ }+ let usage = Usage {+ input_tokens: record+ .prompt_tokens+ .and_then(|input| input.checked_sub(read)?.checked_sub(written)),+ output_tokens: record.completion_tokens,+ cache,+ ..Usage::default()+ };+ value["cost"] = serde_json::to_value(pricing.estimate(&record.model, &usage))?;+ }+ serde_json::to_writer(&mut output, &value)?;+ output.write_all(b"\n")?;+ }+ output.flush()?;+ std::fs::write(destination.join("prices.json"), rates)?;+ std::fs::write(+ destination.join("menubar.toml"),+ toml::to_string_pretty(&config)?,+ )?;+ Ok(())+ })();+ if result.is_err() {+ std::fs::remove_dir_all(&destination)?;+ }+ result+}++fn main() -> std::process::ExitCode {+ let args: Vec<_> = std::env::args_os().skip(1).collect();+ if args.len() != 3 {+ eprintln!(+ "Usage: switchyard-costs OLD_MENUBAR_TOML ROUTING_JSONL NEW_DIRECTORY\nStop the server and menubar first. Outputs use the rates in the old settings."+ );+ return std::process::ExitCode::FAILURE;+ }+ match migrate(+ Path::new(&args[0]),+ Path::new(&args[1]),+ Path::new(&args[2]),+ ) {+ Ok(()) => std::process::ExitCode::SUCCESS,+ Err(error) => {+ eprintln!("switchyard-costs: {error}");+ std::process::ExitCode::FAILURE+ }+ }+}diff --git a/crates/switchyard-menubar/Cargo.toml b/crates/switchyard-menubar/Cargo.tomlindex d12551f0d..b839ad9c9 100644--- a/crates/switchyard-menubar/Cargo.toml+++ b/crates/switchyard-menubar/Cargo.toml@@ -14,6 +14,8 @@ rust-version.workspace = truepublish = false[dependencies]+switchyard-costs.workspace = true+switchyard-protocol.workspace = truechrono = { version = "0.4", default-features = false, features = ["std", "clock"] }serde.workspace = trueserde_json.workspace = truediff --git a/crates/switchyard-menubar/README.md b/crates/switchyard-menubar/README.mdindex 4cc99d208..8525a5878 100644--- a/crates/switchyard-menubar/README.md+++ b/crates/switchyard-menubar/README.md@@ -36,55 +36,69 @@ routing_log = "~/.switchyard/routing.jsonl"baseline_model = "gpt-5.6-sol"refresh_seconds = 30-[prices."gpt-5.6-sol"]-input_per_mtok = 1.25-cached_input_per_mtok = 0.125-output_per_mtok = 10.0+pricing_file = "~/.switchyard/prices.json"```+The server's `--pricing-file` and the menubar's `pricing_file` use the same JSON file.+Key each entry by the model ID in the routing log. Rates are in USD per million tokens:++```json+{"gpt-5.6-sol":{"input":1.25,"cache_read":0.125,"cache_write":1.25,"output":10}}+```++These example rates preserve the installer's previous assumptions. Replace them with+what you pay. The [pi guide](../../docs/integrations/pi.md#estimate-the-cost) explains+cache-write rates and how to add other models. Rates load at startup; restart both+applications after editing them.+## How savings are computed-The server writes one JSONL record per call, with the model that answered and-its token counts.--- **Actual** is every call priced at the model that served it, including- Switchyard's own classifier calls.-- **Baseline** is the caller-facing calls only, priced as if each had used- `baseline_model`. Classifier calls have no baseline counterpart, because- without Switchyard they would not happen.--Savings are the difference, so routing overhead counts against the figure and-a bad day shows a negative number. Dollar figures stay hidden until every-model seen has a price, so a partial table cannot mislead.--Codex's "Approve for me" reviews count too. The server config that the-installer writes sends them to `codex-auto-review` on the ChatGPT backend. With-a ChatGPT login, Codex sends these reviews to `codex-auto-review` even without-Switchyard, so the installer prices that model at the `baseline_model` rates.-Each review then adds the same amount to the actual cost and to the baseline,-so reviews do not change the dollar amount saved. They do lower the percentage-a little, because the baseline grows. If you change `baseline_model`, update-this price to match.--The menu bar needs a price for every model ID that the routing log records.-The macOS installer includes prices for the default models, including-`gpt-6-luna`, `gpt-6-sol`, and `gpt-6.1-sol`. Existing `menubar.toml` files are-preserved during reinstall, so add any missing model prices to that file by-hand.--Codex's "Approve for me" reviews need a price too. Use the reviewer target's-`id`, which is the model ID recorded in the routing log. The default reviewer-route uses `codex-auto-review`. With a ChatGPT login, these reviews also happen-without Switchyard. Add that ID to the price table at the `baseline_model`-rates, including the cached input rate. Each review then adds the same cost to-actual and baseline. Dollars saved stay the same; the percentage drops as the-baseline grows. Update this price when you change `baseline_model`.--If a reviewer uses another model ID, add a price for that ID. Any period with-an unpriced model hides its savings and shows the missing-price hint.--These are list-price estimates. On a ChatGPT login there is no per-token bill,-so read them as "what this traffic would have cost at API rates".+Actual spending is the sum of the costs saved in the routing log, including routing+calls. Changing prices does not change saved costs. The baseline is an estimate of+what the answer traffic would have cost using `baseline_model`; it excludes routing+calls. The server and menubar share the code for cost totals and token pricing, including+cache-read and cache-write rates.+The baseline assumes the same token and cache counts as the observed answer traffic.++The menubar shows estimated savings only when actual costs and the baseline estimate+are complete. A recorded `cost: null` keeps spending unknown. A missing baseline rate+or required one-hour cache-write rate keeps the baseline unknown. Missing or invalid answer+usage also keeps the baseline unknown. Token counts still+appear when savings cannot be calculated. The log can be read while the server is offline.++Codex's `codex-auto-review` model keeps the installer's baseline rates by default.+Update that entry when changing `baseline_model` if you want those reviews to have+the same estimated cost with and without Switchyard. ChatGPT subscription traffic+may have no per-token bill; token-rate estimates are comparisons at the supplied rates.++## Migrate older settings and logs++Inline `[prices]` settings and log records without a `cost` field require migration.+Stop the server and menubar before migrating. From the repository root, run:++```sh+cargo run -p switchyard-costs -- \+ ~/.switchyard/menubar.toml ~/.switchyard/routing.jsonl ~/.switchyard/migrated-costs+```++The destination directory must not exist. The command writes `menubar.toml`,+`prices.json`, and `routing.jsonl` there and leaves the originals untouched. It preserves+existing costs, including null, and adds estimates only to records without a cost field.+Missing usage or rates leave the reconstructed cost unknown. Malformed or unfinished+records stop migration; failed migration outputs are removed.++The command uses the rates in the old settings. It cannot recover historical provider+charges or historical prices. It maps `input_per_mtok` to `input` and `cache_write`,+`output_per_mtok` to `output`, and `cached_input_per_mtok` to `cache_read` (or the input+rate when omitted). This preserves the old cache-write assumption. Review those rates+for your providers; add `cache_write_1h` when needed. Reconstruction is always marked+`estimated`, even when it uses historical prices.++Review the generated files, then start the menubar with the generated settings path.+Those settings point at the generated log and pricing file. Update the server's+LaunchAgent to use those same paths for `--routing-log-file` and `--pricing-file` before+restarting it. Keep the originals as backups. Reinstalling preserves existing files;+it does not migrate them.## Codex profilesdiff --git a/crates/switchyard-menubar/src/app.rs b/crates/switchyard-menubar/src/app.rsindex 158411f40..3695ce9b0 100644--- a/crates/switchyard-menubar/src/app.rs+++ b/crates/switchyard-menubar/src/app.rs@@ -12,15 +12,10 @@ use crate::summary::{Row, build};/// Reads new log entries, probes the server, and builds the menu body.pub fn refresh(config: &Config, log: &mut rollup::Reader) -> Vec<Row> {- let usage = log- .read(&config.routing_log, Local::now().date_naive())- .unwrap_or_else(|error| {- eprintln!(- "switchyard-menubar: read {}: {error}",- config.routing_log.display()- );- rollup::Usage::default()- });+ let usage = match log.read(&config.routing_log, Local::now().date_naive()) {+ Ok(usage) => usage,+ Err(error) => return vec![Row::Label(format!("Cannot read routing log: {error}"))],+ };build(probe(&config.server_url), &usage, config)}@@ -36,7 +31,7 @@ mod tests {std::fs::write(&path,format!(- r#"{{"ts":"{ts}","model":"luna","tier":"","prompt_tokens":1000,"cached_tokens":0,"completion_tokens":100,"reasoning_tokens":0}}"#+ r#"{{"ts":"{ts}","model":"luna","tier":"","prompt_tokens":1000,"cached_tokens":0,"completion_tokens":100,"reasoning_tokens":0,"cost":null}}"#) + "\n",).expect("write log");diff --git a/crates/switchyard-menubar/src/config.rs b/crates/switchyard-menubar/src/config.rsindex 8b29b4b6a..cebc785bf 100644--- a/crates/switchyard-menubar/src/config.rs+++ b/crates/switchyard-menubar/src/config.rs@@ -7,7 +7,7 @@ use std::path::{Path, PathBuf};use serde::Deserialize;-use crate::pricing::PriceTable;+use switchyard_costs::Pricing;/// Everything the menu bar needs to find the server and price its traffic./// Missing keys fall back to [`Config::default`].@@ -26,8 +26,11 @@ pub struct Config {pub refresh_seconds: u64,/// Model the traffic is assumed to have used without Switchyard.pub baseline_model: String,- /// Per-model rates. Dollar figures are hidden while a seen model is absent.- pub prices: PriceTable,+ /// The shared JSON file supplies rates for the baseline estimate.+ pub pricing_file: Option<PathBuf>,+ /// Rates load once when the settings are read.+ #[serde(skip)]+ pub prices: Pricing,}impl Default for Config {@@ -39,7 +42,8 @@ impl Default for Config {launchd_label: "com.nvidia.switchyard.server".to_string(),refresh_seconds: 30,baseline_model: "gpt-5.6-sol".to_string(),- prices: PriceTable::new(),+ pricing_file: None,+ prices: Pricing::default(),}}}@@ -53,6 +57,11 @@ impl Config {Err(error) if error.kind() == std::io::ErrorKind::NotFound => Self::default(),Err(error) => return Err(format!("read {}: {error}", path.display())),};+ if let Some(path) = config.pricing_file.as_ref() {+ let path = expand_home(path);+ config.prices = Pricing::load(&path)?;+ config.pricing_file = Some(path);+ }config.routing_log = expand_home(&config.routing_log);config.config_file = expand_home(&config.config_file);Ok(config)@@ -100,10 +109,6 @@ mod tests {server_url = "http://127.0.0.1:9000"baseline_model = "sol"-[prices.luna]-input_per_mtok = 0.25-cached_input_per_mtok = 0.025-output_per_mtok = 2.0"#,).expect("write settings");@@ -112,7 +117,7 @@ output_per_mtok = 2.0assert_eq!(config.server_url, "http://127.0.0.1:9000");assert_eq!(config.baseline_model, "sol");- assert_eq!(config.prices["luna"].cached_input_per_mtok, Some(0.025));+ assert!(config.pricing_file.is_none());assert_eq!(config.refresh_seconds, 30, "unset keys keep their default");}@@ -123,7 +128,7 @@ output_per_mtok = 2.0let config = Config::load(&dir.path().join("absent.toml")).expect("load");assert_eq!(config.server_url, "http://127.0.0.1:4123");- assert!(config.prices.is_empty());+ assert!(config.pricing_file.is_none());let empty = dir.path().join("empty.toml");std::fs::write(&empty, "").expect("write settings");let loaded = Config::load(&empty).expect("load empty settings");diff --git a/crates/switchyard-menubar/src/pricing.rs b/crates/switchyard-menubar/src/pricing.rsindex e551bdd71..e2cf1c98d 100644--- a/crates/switchyard-menubar/src/pricing.rs+++ b/crates/switchyard-menubar/src/pricing.rs@@ -4,37 +4,10 @@//! Counterfactual cost: what routed traffic cost against what it would have//! cost had every call gone to one capable model.-use std::collections::BTreeMap;+use switchyard_costs::{CostTotals, Pricing};+use switchyard_protocol::{InputCacheUsage, Usage};-use serde::Deserialize;--use crate::rollup::{ModelTokens, Totals};--const PER_MILLION: f64 = 1_000_000.0;--/// Per-million-token rates for one model.-#[derive(Clone, Copy, Debug, Deserialize, PartialEq)]-pub struct ModelPrice {- pub input_per_mtok: f64,- /// Rate for cache reads. Falls back to the full input rate.- #[serde(default)]- pub cached_input_per_mtok: Option<f64>,- /// Rate for generated tokens, reasoning included.- pub output_per_mtok: f64,-}--impl ModelPrice {- fn cost(&self, tokens: &ModelTokens) -> f64 {- let cached_rate = self.cached_input_per_mtok.unwrap_or(self.input_per_mtok);- (tokens.input as f64 * self.input_per_mtok- + tokens.cached_input as f64 * cached_rate- + tokens.output as f64 * self.output_per_mtok)- / PER_MILLION- }-}--/// Rates keyed by the model id the server records in the routing log.-pub type PriceTable = BTreeMap<String, ModelPrice>;+use crate::rollup::Totals;/// What the traffic cost, and what it would have cost without routing.#[derive(Clone, Copy, Debug, PartialEq)]@@ -59,44 +32,52 @@ impl Savings {/// Prices a period against the baseline model.///-/// Returns `None` when any model seen has no entry in the table, since a-/// partial bill would understate cost and overstate savings.-pub fn estimate(totals: &Totals, prices: &PriceTable, baseline_model: &str) -> Option<Savings> {- let baseline_price = prices.get(baseline_model)?;-- let mut actual = 0.0;- let mut baseline = 0.0;- for (model, tokens) in &totals.routed {- actual += prices.get(model)?.cost(tokens);- baseline += baseline_price.cost(tokens);+/// Returns `None` when recorded spending or the baseline estimate is incomplete.+pub fn estimate(totals: &Totals, prices: &Pricing, baseline_model: &str) -> Option<Savings> {+ if totals.baseline_unknown {+ return None;}- // Classifier calls are Switchyard's own cost. They have no baseline- // counterpart, so they only ever reduce the savings figure.- for (model, tokens) in &totals.classifier {- actual += prices.get(model)?.cost(tokens);+ let mut baseline = CostTotals::default();+ for tokens in totals.routed.values() {+ let usage = Usage {+ input_tokens: Some(tokens.input),+ output_tokens: Some(tokens.output),+ cache: Some(Box::new(InputCacheUsage {+ cached_input_tokens: Some(tokens.cached_input),+ cache_creation_input_tokens: Some(tokens.cache_write),+ cache_creation_1h_input_tokens: Some(tokens.cache_write_1h),+ })),+ ..Usage::default()+ };+ baseline.add(prices.estimate(baseline_model, &usage));}-- Some(Savings { actual, baseline })+ Some(Savings {+ actual: totals.cost.total_usd?,+ baseline: baseline.total_usd?,+ })}#[cfg(test)]mod tests {use super::*;-- fn price(input: f64, output: f64) -> ModelPrice {- ModelPrice {- input_per_mtok: input,- cached_input_per_mtok: None,- output_per_mtok: output,- }+ use crate::rollup::ModelTokens;++ fn table() -> Pricing {+ Pricing::parse(+ br#"{+ "sol":{"input":1.25,"cache_read":1.25,"cache_write":1.25,"output":10},+ "luna":{"input":0.25,"cache_read":0.25,"cache_write":0.25,"output":2},+ "terra":{"input":0.05,"cache_read":0.05,"cache_write":0.05,"output":0.4}+ }"#,+ )+ .expect("rates")}- fn table() -> PriceTable {- PriceTable::from([- ("sol".to_string(), price(1.25, 10.0)),- ("luna".to_string(), price(0.25, 2.0)),- ("terra".to_string(), price(0.05, 0.4)),- ])+ fn record_cost(totals: &mut Totals, usd: f64) {+ totals.cost.add(Some(switchyard_costs::RecordedCost {+ usd,+ source: switchyard_costs::CostSource::Estimated,+ }));}fn tokens(input: u64, output: u64) -> ModelTokens {@@ -105,6 +86,7 @@ mod tests {input,cached_input: 0,output,+ ..ModelTokens::default()}}@@ -115,6 +97,7 @@ mod tests {.routed.insert("luna".to_string(), tokens(1_000_000, 100_000));+ record_cost(&mut totals, 0.45);let savings = estimate(&totals, &table(), "sol").expect("priced");assert!((savings.actual - 0.45).abs() < 1e-9);@@ -128,11 +111,13 @@ mod tests {totals.routed.insert("luna".to_string(), tokens(1_000_000, 100_000));+ record_cost(&mut totals, 0.45);let before = estimate(&totals, &table(), "sol").expect("priced");totals.classifier.insert("terra".to_string(), tokens(1_000_000, 10_000));+ record_cost(&mut totals, 0.054);let after = estimate(&totals, &table(), "sol").expect("priced");assert_eq!(after.baseline, before.baseline);@@ -141,14 +126,9 @@ mod tests {#[test]fn cache_reads_use_the_cached_rate() {- let prices = PriceTable::from([(- "sol".to_string(),- ModelPrice {- input_per_mtok: 1.0,- cached_input_per_mtok: Some(0.1),- output_per_mtok: 0.0,- },- )]);+ let prices =+ Pricing::parse(br#"{"sol":{"input":1,"cache_read":0.1,"cache_write":1,"output":0}}"#)+ .expect("rates");let mut totals = Totals::default();totals.routed.insert("sol".to_string(),@@ -157,23 +137,26 @@ mod tests {input: 1_000_000,cached_input: 1_000_000,output: 0,+ ..ModelTokens::default()},);+ record_cost(&mut totals, 1.1);let savings = estimate(&totals, &prices, "sol").expect("priced");assert!((savings.actual - 1.1).abs() < 1e-9);}#[test]- fn an_unpriced_model_suppresses_the_estimate() {+ fn an_unknown_cost_suppresses_savings() {let mut totals = Totals::default();totals.routed.insert("unknown".to_string(), tokens(1_000, 100));+ totals.cost.add(None);assert!(estimate(&totals, &table(), "sol").is_none());- assert!(estimate(&Totals::default(), &table(), "absent").is_none());+ assert!(estimate(&totals, &table(), "absent").is_none());}#[test]diff --git a/crates/switchyard-menubar/src/rollup.rs b/crates/switchyard-menubar/src/rollup.rsindex c05a34780..0641ccb73 100644--- a/crates/switchyard-menubar/src/rollup.rs+++ b/crates/switchyard-menubar/src/rollup.rs@@ -10,6 +10,7 @@ use std::path::Path;use chrono::{DateTime, Local, NaiveDate};use serde::Deserialize;+use switchyard_costs::{CostTotals, RecordedCost};/// Tier value the server writes for classifier and judge calls.const CLASSIFIER_TIER: &str = "classifier";@@ -18,19 +19,22 @@ const CLASSIFIER_TIER: &str = "classifier";#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]pub struct ModelTokens {pub requests: u64,- /// Input tokens billed at the full rate. Cache writes bill at that rate- /// too, so they are counted here rather than tracked separately.+ /// Input tokens exclude cache reads and writes.pub input: u64,/// Input tokens served from the provider cache.pub cached_input: u64,/// Generated tokens, including reasoning.pub output: u64,+ /// Cache-write tokens are separate from uncached input.+ pub cache_write: u64,+ /// One-hour writes are a subset of all cache writes.+ pub cache_write_1h: u64,}impl ModelTokens {/// Every token the provider counted, cached reads included.pub fn total(&self) -> u64 {- self.input + self.cached_input + self.output+ self.input + self.cached_input + self.cache_write + self.output}fn add(&mut self, other: &Self) {@@ -38,6 +42,8 @@ impl ModelTokens {self.input += other.input;self.cached_input += other.cached_input;self.output += other.output;+ self.cache_write += other.cache_write;+ self.cache_write_1h += other.cache_write_1h;}}@@ -45,8 +51,12 @@ impl ModelTokens {////// The split matters for savings: without Switchyard the `routed` calls would/// still have happened, but the `classifier` calls would not exist at all.-#[derive(Clone, Debug, Default, PartialEq, Eq)]+#[derive(Clone, Debug, Default, PartialEq)]pub struct Totals {+ /// Actual spending comes only from the saved cost of each call.+ pub cost: CostTotals,+ /// Missing or invalid answer usage keeps savings unknown after totals are merged.+ pub baseline_unknown: bool,/// Calls that served the caller, keyed by the model that answered.pub routed: BTreeMap<String, ModelTokens>,/// Switchyard's own classifier and judge calls, keyed by model.@@ -55,6 +65,10 @@ pub struct Totals {impl Totals {fn add(&mut self, record: &Record) {+ self.cost.add(record.cost);+ if record.tier != CLASSIFIER_TIER {+ self.baseline_unknown |= !record.has_baseline_usage();+ }let bucket = if record.tier == CLASSIFIER_TIER {&mut self.classifier} else {@@ -67,6 +81,8 @@ impl Totals {}fn merge(&mut self, other: &Self) {+ self.cost.merge(&other.cost);+ self.baseline_unknown |= other.baseline_unknown;for (model, tokens) in &other.routed {self.routed.entry(model.clone()).or_default().add(tokens);}@@ -113,22 +129,48 @@ struct Record {#[serde(default)]tier: String,#[serde(default)]- prompt_tokens: u64,+ prompt_tokens: Option<u64>,+ #[serde(default)]+ cached_tokens: Option<u64>,+ #[serde(default)]+ completion_tokens: Option<u64>,#[serde(default)]- cached_tokens: u64,+ cache_creation_tokens: Option<u64>,#[serde(default)]- completion_tokens: u64,+ cache_creation_1h_tokens: Option<u64>,+ // A missing field requires migration; explicit null is a valid unknown cost.+ #[serde(deserialize_with = "Option::deserialize")]+ cost: Option<RecordedCost>,}impl Record {+ // Each record must be valid before aggregation can conceal inconsistent cache counts.+ fn has_baseline_usage(&self) -> bool {+ let Some(input) = self.prompt_tokens else {+ return false;+ };+ self.completion_tokens.is_some()+ && input+ .checked_sub(self.cached_tokens.unwrap_or(0))+ .and_then(|input| input.checked_sub(self.cache_creation_tokens.unwrap_or(0)))+ .is_some()+ && self.cache_creation_1h_tokens.unwrap_or(0) <= self.cache_creation_tokens.unwrap_or(0)+ }+fn tokens(&self) -> ModelTokens {ModelTokens {requests: 1,// `prompt_tokens` is the whole input, cache reads included.- input: self.prompt_tokens.saturating_sub(self.cached_tokens),- cached_input: self.cached_tokens,+ input: self+ .prompt_tokens+ .unwrap_or(0)+ .saturating_sub(self.cached_tokens.unwrap_or(0))+ .saturating_sub(self.cache_creation_tokens.unwrap_or(0)),+ cached_input: self.cached_tokens.unwrap_or(0),// Completion tokens already include reasoning tokens.- output: self.completion_tokens,+ output: self.completion_tokens.unwrap_or(0),+ cache_write: self.cache_creation_tokens.unwrap_or(0),+ cache_write_1h: self.cache_creation_1h_tokens.unwrap_or(0),}}}@@ -176,10 +218,22 @@ impl Reader {if line.last() != Some(&b'\n') {break;}- self.offset += bytes as u64;- let Ok(record) = serde_json::from_slice::<Record>(&line) else {- continue;+ let record = match serde_json::from_slice::<Record>(&line) {+ Ok(record) => record,+ Err(error) if error.is_data() => {+ return Err(std::io::Error::new(+ std::io::ErrorKind::InvalidData,+ format!(+ "invalid routing record: {error}; migrate older logs before reading them"+ ),+ ));+ }+ Err(_) => {+ self.offset += bytes as u64;+ continue;+ }};+ self.offset += bytes as u64;let Ok(ts) = DateTime::parse_from_rfc3339(&record.ts) else {continue;};@@ -232,7 +286,7 @@ mod tests {fn line(local_ts: &str, model: &str, tier: &str, prompt: u64, completion: u64) -> String {let offset = Local::now().offset().to_string();format!(- r#"{{"ts":"{local_ts}{offset}","model":"{model}","tier":"{tier}","prompt_tokens":{prompt},"cached_tokens":0,"completion_tokens":{completion},"reasoning_tokens":0}}"#+ r#"{{"ts":"{local_ts}{offset}","model":"{model}","tier":"{tier}","prompt_tokens":{prompt},"cached_tokens":0,"completion_tokens":{completion},"reasoning_tokens":0,"cost":null}}"#)}@@ -266,7 +320,7 @@ mod tests {#[test]fn separates_cache_reads_from_full_price_input() {let record: Record = serde_json::from_str(- r#"{"ts":"2026-09-28T10:00:00.000Z","model":"m","prompt_tokens":1000,"cached_tokens":600,"completion_tokens":50,"reasoning_tokens":25}"#,+ r#"{"ts":"2026-09-28T10:00:00.000Z","model":"m","prompt_tokens":1000,"cached_tokens":600,"completion_tokens":50,"reasoning_tokens":25,"cost":null}"#,).expect("parse record");let tokens = record.tokens();@@ -279,27 +333,39 @@ mod tests {#[test]fn counts_completion_tokens_once_and_prices_them() {let record: Record = serde_json::from_str(- r#"{"ts":"2026-09-28T10:00:00.000Z","model":"m","prompt_tokens":12,"cached_tokens":0,"completion_tokens":5,"reasoning_tokens":3}"#,+ r#"{"ts":"2026-09-28T10:00:00.000Z","model":"m","prompt_tokens":12,"cached_tokens":0,"cache_creation_tokens":10,"completion_tokens":5,"reasoning_tokens":3,"cost":{"usd":0.000022,"source":"provider_reported"}}"#,).expect("parse record");let tokens = record.tokens();- assert_eq!(tokens.input, 12);+ assert_eq!(tokens.input, 2);assert_eq!(tokens.output, 5);assert_eq!(tokens.total(), 17);let mut totals = Totals::default();totals.add(&record);- let prices = crate::pricing::PriceTable::from([(- "m".to_string(),- crate::pricing::ModelPrice {- input_per_mtok: 1.0,- cached_input_per_mtok: None,- output_per_mtok: 2.0,- },- )]);+ let prices = switchyard_costs::Pricing::parse(+ br#"{"m":{"input":1,"cache_read":1,"cache_write":1,"cache_write_1h":3,"output":2}}"#,+ )+ .expect("rates");let savings = crate::pricing::estimate(&totals, &prices, "m").expect("priced");assert!((savings.actual - 0.000022).abs() < 1e-12);+ // A known zero charge still needs valid usage for the hypothetical baseline.+ for fields in [+ r#""prompt_tokens":null,"completion_tokens":5"#,+ r#""completion_tokens":5"#,+ r#""prompt_tokens":2,"cached_tokens":3,"completion_tokens":5"#,+ r#""prompt_tokens":20,"cache_creation_tokens":2,"cache_creation_1h_tokens":3,"completion_tokens":5"#,+ ] {+ let record: Record = serde_json::from_str(&format!(+ r#"{{"ts":"2026-09-28T10:00:00Z","model":"m",{fields},"cost":{{"usd":0,"source":"provider_reported"}}}}"#+ )).expect("record with incomplete baseline usage");+ let mut incomplete = Totals::default();+ incomplete.add(&record);+ incomplete.merge(&totals);+ assert_eq!(incomplete.cost.total_usd, Some(0.000022));+ assert!(crate::pricing::estimate(&incomplete, &prices, "m").is_none());+ }}#[test]diff --git a/crates/switchyard-menubar/src/summary.rs b/crates/switchyard-menubar/src/summary.rsindex 9c18cb6e2..024d42f01 100644--- a/crates/switchyard-menubar/src/summary.rs+++ b/crates/switchyard-menubar/src/summary.rs@@ -51,7 +51,9 @@ pub fn build(status: ServerStatus, usage: &Usage, config: &Config) -> Vec<Row> {if estimate(&usage.week, &config.prices, &config.baseline_model).is_none() {rows.push(Row::Separator);- rows.push(label("Add prices to menubar.toml to see savings"));+ rows.push(label(+ "Savings need complete recorded costs and baseline rates in prices.json",+ ));}rows}@@ -64,7 +66,7 @@ fn period(name: &str, totals: &Totals, config: &Config) -> Vec<Row> {tokens(totals.tokens())))];if let Some(savings) = estimate(totals, &config.prices, &config.baseline_model) {- rows.push(label(format!(" Saved {}", saved(&savings))));+ rows.push(label(format!(" Estimated savings {}", saved(&savings))));}rows}@@ -123,8 +125,8 @@ fn money(value: f64) -> String {#[cfg(test)]mod tests {use super::*;- use crate::pricing::{ModelPrice, PriceTable};use crate::rollup::ModelTokens;+ use switchyard_costs::{CostSource, Pricing, RecordedCost};fn counted(input: u64, output: u64, requests: u64) -> ModelTokens {ModelTokens {@@ -132,10 +134,11 @@ mod tests {input,cached_input: 0,output,+ ..ModelTokens::default()}}- fn config(prices: PriceTable) -> Config {+ fn config(prices: Pricing) -> Config {Config {baseline_model: "sol".to_string(),prices,@@ -143,25 +146,8 @@ mod tests {}}- fn priced() -> PriceTable {- PriceTable::from([- (- "sol".to_string(),- ModelPrice {- input_per_mtok: 1.25,- cached_input_per_mtok: None,- output_per_mtok: 10.0,- },- ),- (- "luna".to_string(),- ModelPrice {- input_per_mtok: 0.25,- cached_input_per_mtok: None,- output_per_mtok: 2.0,- },- ),- ])+ fn priced() -> Pricing {+ Pricing::parse(br#"{"sol":{"input":1.25,"cache_read":1.25,"cache_write":1.25,"output":10},"luna":{"input":0.25,"cache_read":0.25,"cache_write":0.25,"output":2}}"#).expect("rates")}fn labels(rows: &[Row]) -> Vec<String> {@@ -187,6 +173,14 @@ mod tests {week.routed.insert("luna".to_string(), counted(4_000_000, 400_000, 12));+ today.cost.add(Some(RecordedCost {+ usd: 0.45,+ source: CostSource::Estimated,+ }));+ week.cost.add(Some(RecordedCost {+ usd: 1.80,+ source: CostSource::Estimated,+ }));let rows = build(ServerStatus::Running,&usage_of(today, week),@@ -196,9 +190,9 @@ mod tests {assert_eq!(labels[0], "Server: running · http://127.0.0.1:4123");assert_eq!(labels[1], "Today — 3 requests · 1.1M tokens");- assert_eq!(labels[2], " Saved $1.80 (80% of $2.25)");+ assert_eq!(labels[2], " Estimated savings $1.80 (80% of $2.25)");assert_eq!(labels[3], "This week — 12 requests · 4.4M tokens");- assert_eq!(labels[4], " Saved $7.20 (80% of $9.00)");+ assert_eq!(labels[4], " Estimated savings $7.20 (80% of $9.00)");}#[test]@@ -209,19 +203,20 @@ mod tests {week.classifier.insert("sol".to_string(), counted(100, 0, 1));- for prices in [- PriceTable::new(),- PriceTable::from([("sol".to_string(), priced()["sol"])]),- PriceTable::from([("luna".to_string(), priced()["luna"])]),- ] {+ week.cost.add(None);+ for prices in [Pricing::default(), priced()] {let rows = build(ServerStatus::Running,&usage_of(week.clone(), week.clone()),&config(prices),);let labels = labels(&rows);- assert!(labels.iter().all(|label| !label.contains("Saved")));- assert!(labels.iter().any(|label| label.contains("menubar.toml")));+ assert!(+ labels+ .iter()+ .all(|label| !label.contains("Estimated savings"))+ );+ assert!(labels.iter().any(|label| label.contains("costs")));}}@@ -263,13 +258,17 @@ mod tests {.classifier.insert("sol".to_string(), counted(1_000_000, 0, 1));+ today.cost.add(Some(RecordedCost {+ usd: 1.25125,+ source: CostSource::Estimated,+ }));let rows = build(ServerStatus::Running,&usage_of(today.clone(), today),&config(priced()),);- assert!(labels(&rows)[2].starts_with(" Saved -$1.25"));+ assert!(labels(&rows)[2].starts_with(" Estimated savings -$1.25"));}#[test]diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rsindex 087034832..e561ff8c9 100644--- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs+++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs@@ -1415,6 +1415,7 @@ mod tests {output_tokens: Some(7),total_tokens: Some(23),reasoning_tokens: Some(5),+ provider_cost: None,}),};diff --git a/crates/switchyard-server/Cargo.toml b/crates/switchyard-server/Cargo.tomlindex 2ef504d7a..3601dd083 100644--- a/crates/switchyard-server/Cargo.toml+++ b/crates/switchyard-server/Cargo.toml@@ -20,6 +20,7 @@ publish = ["crates-io"]prefill-router = ["switchyard-runner/prefill-router"][dependencies]+switchyard-costs.workspace = trueasync-stream.workspace = trueasync-trait.workspace = trueaxum = "0.8"diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.mdindex 08335197f..aa454a460 100644--- a/crates/switchyard-server/README.md+++ b/crates/switchyard-server/README.md@@ -124,6 +124,22 @@ served model. The legacy `proxy_x_session_id` remains a fallback when no normalipresent. The endpoint returns `404` when the session has no records and is not registered whenrouting logging is disabled.+The routing log saves each call's reported charge in USD. Add `--pricing-file prices.json`+to estimate costs that the upstream does not report. The JSON maps served model IDs to rates+in USD per million tokens:++```json+{"upstream/model":{"input":1,"output":5,"cache_read":0.1,"cache_write":2}}+```++Session stats include cost totals. Missing usage or rates leave cost unknown; `known_usd`+keeps the priced subtotal and `total_usd` is null when any cost is unknown. Rates load once+at startup, and saved costs do not change when prices change.++The [pi guide](../../docs/integrations/pi.md#estimate-the-cost) explains how to supply+rates for your own models. The menubar reads the same file through its `pricing_file`+setting; its actual spending comes from recorded costs.+Clients can send `x-switchyard-origin: codex-cli` (or another client label) to include an`origin` field in each routing record. Missing, empty, or non-text header values produce`"origin": null`. The value is supplied by the caller; it is not inferred from `User-Agent`.diff --git a/crates/switchyard-server/src/cli.rs b/crates/switchyard-server/src/cli.rsindex 231d4e2bc..969445792 100644--- a/crates/switchyard-server/src/cli.rs+++ b/crates/switchyard-server/src/cli.rs@@ -52,6 +52,10 @@ pub(crate) struct ServerArgs {#[arg(long, value_name = "PATH")]routing_log_file: Option<PathBuf>,+ /// This JSON file maps served model IDs to rates in USD per million tokens.+ #[arg(long, value_name = "PATH", requires = "routing_log_file")]+ pricing_file: Option<PathBuf>,+/// TLS certificate path in PEM format.#[arg(long, requires = "tls_key")]tls_cert: Option<PathBuf>,@@ -72,6 +76,9 @@ impl ServerArgs {if let Some(path) = self.routing_log_file {state = state.with_routing_log(path)?;}+ if let Some(path) = self.pricing_file {+ state = state.with_pricing_file(path)?;+ }let tls = match (self.tls_cert, self.tls_key) {(Some(cert), Some(key)) => {if !cert.exists() || !key.exists() {diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rsindex 9e3ac630a..4fed60a30 100644--- a/crates/switchyard-server/src/lib.rs+++ b/crates/switchyard-server/src/lib.rs@@ -237,6 +237,17 @@ impl ServerState {Ok(self)}+ /// This method loads rates in USD per million tokens for saved routing records.+ pub fn with_pricing_file(self, path: impl AsRef<std::path::Path>) -> ServerResult<Self> {+ let prices = switchyard_costs::Pricing::load(path.as_ref()).map_err(ServerError::new)?;+ let log = self+ .routing_log+ .as_ref()+ .ok_or_else(|| ServerError::new("pricing requires a routing log"))?;+ log.writer.lock().pricing = prices;+ Ok(self)+ }+/// Returns the route model IDs served by the configured algorithms.pub fn models(&self) -> impl Iterator<Item = &str> {self.runner.models().map(|model| model.id.as_str())@@ -429,13 +440,11 @@ struct RequestStart(Instant);/// Routing-log marker distinguishing classifier and judge calls from answer calls.const CLASSIFIER_TIER: &str = "classifier";-/// Maps answer-call observations to backend stats, routing calls to classifier/judge-/// stats, and records routing overhead once the algorithm run completes.+/// This observer updates backend stats and records classifier and judge calls.///-/// Successful classifier/judge calls are also appended to `classifier_log` when one is-/// configured, so per-session routing snapshots account for judge token overhead. Routed-/// calls stay off this path: the served call is logged with its terminal usage in-/// [`usage_metrics::observe`], which would make a second append here a double count.+/// The routing log also records failed answer calls with unknown cost. Successful+/// answer calls are recorded by [`usage_metrics::observe`] with their terminal usage;+/// recording them here would count each answer twice.fn stats_observer(stats: StatsAccumulator,classifier_log: Option<(SharedRoutingLog, routing_log::RoutingLogContext)>,@@ -447,22 +456,28 @@ fn stats_observer(if call.is_success {stats.record_success(&call.selected_model, latency_ms);} else {+ if let Some((log, context)) = classifier_log.as_ref() {+ log.append(+ context.clone(),+ &call.selected_model,+ None,+ &Usage::default(),+ );+ }stats.record_error(&call.selected_model);}}RunObservation::LlmCall(call) | RunObservation::DecisionCall(call) => {let latency_ms = call.duration.as_secs_f64() * 1_000.0;+ if let Some((log, context)) = classifier_log.as_ref() {+ log.append(+ context.clone(),+ &call.selected_model,+ Some(CLASSIFIER_TIER),+ call.usage.as_ref().unwrap_or(&Usage::default()),+ );+ }if call.is_success {- if let (Some((log, context)), Some(usage)) =- (classifier_log.as_ref(), call.usage.as_ref())- {- log.append(- context.clone(),- &call.selected_model,- Some(CLASSIFIER_TIER),- usage,- );- }stats.record_classifier_success(call.selected_model,call.usage.as_ref().map(usage_metrics::token_usage),diff --git a/crates/switchyard-server/src/routing_log.rs b/crates/switchyard-server/src/routing_log.rsindex 674db1b6c..01dddfb6b 100644--- a/crates/switchyard-server/src/routing_log.rs+++ b/crates/switchyard-server/src/routing_log.rs@@ -16,6 +16,7 @@ use switchyard_protocol::{Metadata, ModelId, Usage};use crate::usage_metrics::token_usage;use crate::{ServerError, ServerResult};+use switchyard_costs::{CostTotals, Pricing, RecordedCost};const LEGACY_SESSION_ID_HEADER: &str = "proxy_x_session_id";const ORIGIN_HEADER: &str = "x-switchyard-origin";@@ -23,7 +24,10 @@ const TASK_HEADER: &str = "x-switchyard-intake-task";const TRIAL_ID_HEADER: &str = "x-switchyard-trial-id";/// Append-only writer for one routing JSONL file.-pub(crate) struct RoutingLog(fs::File);+pub(crate) struct RoutingLog {+ file: fs::File,+ pub(crate) pricing: Pricing,+}impl RoutingLog {pub(crate) fn new(path: impl Into<PathBuf>) -> ServerResult<Self> {@@ -39,7 +43,10 @@ impl RoutingLog {.append(true).open(&path).map_err(|error| routing_log_error(&path, error))?;- Ok(Self(file))+ Ok(Self {+ file,+ pricing: Pricing::default(),+ })}pub(crate) fn append(@@ -49,8 +56,15 @@ impl RoutingLog {tier: Option<&str>,usage: &Usage,) -> std::io::Result<()> {+ let cost = self.pricing.cost(model, usage);+ let cache_creation_1h_tokens = usage+ .cache+ .as_ref()+ .and_then(|cache| cache.cache_creation_1h_input_tokens)+ .unwrap_or(0);let usage = token_usage(usage);let record = RoutingRecord {+ cost,ts: format_rfc3339_millis(SystemTime::now()).to_string().into(),route_id: context.route_id.into(),algorithm: context.algorithm.into(),@@ -64,6 +78,7 @@ impl RoutingLog {prompt_tokens: usage.prompt_tokens,cached_tokens: usage.cached_tokens,cache_creation_tokens: usage.cache_creation_tokens,+ cache_creation_1h_tokens,completion_tokens: usage.completion_tokens,reasoning_tokens: usage.reasoning_tokens,total_tokens: usage.prompt_tokens.saturating_add(usage.completion_tokens),@@ -71,7 +86,7 @@ impl RoutingLog {let mut line = serde_json::to_vec(&record).map_err(std::io::Error::other)?;line.push(b'\n');- self.0.write_all(&line)+ self.file.write_all(&line)}}@@ -82,7 +97,10 @@ pub(crate) fn snapshot() -> std::io::Result<Option<SessionStatsSnapshot>> {let mut reader = BufReader::with_capacity(64 * 1024, fs::File::open(path)?);let mut line = Vec::new();- let mut snapshot = SessionStatsSnapshot::new(session_id);+ let mut snapshot = SessionStatsSnapshot {+ session_id: session_id.to_string(),+ ..Default::default()+ };loop {line.clear();if reader.read_until(b'\n', &mut line)? == 0 {@@ -151,6 +169,8 @@ impl RoutingLogContext {#[derive(Default, Deserialize, Serialize)]#[serde(default)]struct RoutingRecord<'a> {+ #[serde(default)]+ cost: Option<RecordedCost>,ts: Cow<'a, str>,route_id: Cow<'a, str>,algorithm: Cow<'a, str>,@@ -169,15 +189,25 @@ struct RoutingRecord<'a> {prompt_tokens: u64,cached_tokens: u64,cache_creation_tokens: u64,+ /// The part of `cache_creation_tokens` written at the one-hour rate, when reported.+ #[serde(skip_serializing_if = "is_zero")]+ cache_creation_1h_tokens: u64,completion_tokens: u64,reasoning_tokens: u64,total_tokens: u64,}+fn is_zero(value: &u64) -> bool {+ *value == 0+}+/// Session totals returned by the routing stats endpoint.-#[derive(Serialize)]+#[derive(Default, Serialize)]pub(crate) struct SessionStatsSnapshot {session_id: String,+ cost: CostTotals,+ answer_cost: CostTotals,+ routing_cost: CostTotals,total_calls: u64,total_prompt_tokens: u64,total_cached_tokens: u64,@@ -188,6 +218,7 @@ pub(crate) struct SessionStatsSnapshot {#[derive(Default, Serialize)]struct SessionModelStats {+ cost: CostTotals,calls: u64,prompt_tokens: u64,cached_tokens: u64,@@ -196,18 +227,6 @@ struct SessionModelStats {}impl SessionStatsSnapshot {- fn new(session_id: &str) -> Self {- Self {- session_id: session_id.to_string(),- total_calls: 0,- total_prompt_tokens: 0,- total_cached_tokens: 0,- total_cache_creation_tokens: 0,- total_completion_tokens: 0,- models: BTreeMap::new(),- }- }-fn add_record(&mut self, record: &RoutingRecord<'_>, session_id: &str) {if record.session_id.as_deref() != Some(session_id) {return;@@ -217,6 +236,13 @@ impl SessionStatsSnapshot {model => model,};let stats = self.models.entry(ModelId::from(model)).or_default();+ stats.cost.add(record.cost);+ self.cost.add(record.cost);+ if record.tier == "classifier" {+ self.routing_cost.add(record.cost);+ } else {+ self.answer_cost.add(record.cost);+ }stats.calls = stats.calls.saturating_add(1);stats.prompt_tokens = stats.prompt_tokens.saturating_add(record.prompt_tokens);stats.cached_tokens = stats.cached_tokens.saturating_add(record.cached_tokens);diff --git a/crates/switchyard-server/src/usage_metrics.rs b/crates/switchyard-server/src/usage_metrics.rsindex 045f51c86..54c2a1c01 100644--- a/crates/switchyard-server/src/usage_metrics.rs+++ b/crates/switchyard-server/src/usage_metrics.rs@@ -66,6 +66,10 @@ pub(crate) fn observe(}}}+ if failed && !recorded+ && let Some((log, context)) = routing_log.as_ref() {+ log.append(context.clone(), &model, None, &Usage::default());+ }if failed {record_stream_error(&stats, &model);}diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rsindex 315d2470c..d0fad52ab 100644--- a/crates/switchyard-server/tests/server.rs+++ b/crates/switchyard-server/tests/server.rs@@ -4769,10 +4769,16 @@ async fn unavailable_target_fails_over_across_endpoints_and_stops_when_exhausted.lines().map(serde_json::from_str::<Value>).collect::<Result<Vec<_>, _>>()?;- assert_eq!(records.len(), 3);- assert!(records.iter().all(|record| {- record["model"] == "model/strong" && record.get("fallback_reason").is_none()- }));+ assert_eq!(records.len(), 6);+ assert_eq!(+ records+ .iter()+ .filter(|record| {+ record["model"] == "model/strong" && record.get("fallback_reason").is_none()+ })+ .count(),+ 3+ );let previous_call_count = upstream.calls.lock().await.len();let response = send(diff --git a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs b/crates/switchyard-translation/src/codecs/anthropic/buffered.rsindex 88aa8a324..9282c71cc 100644--- a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs+++ b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs@@ -1256,8 +1256,13 @@ fn decode_anthropic_usage(value: Option<&Value>) -> Usage {.and_then(Value::as_u64);let output_tokens = value.get("output_tokens").and_then(Value::as_u64);Usage {+ provider_cost: crate::codecs::common::provider_cost(value),input_tokens,- cache: Usage::cache_details(cached_input_tokens, cache_creation_input_tokens),+ cache: crate::codecs::common::cache_usage_details(+ value,+ cached_input_tokens,+ cache_creation_input_tokens,+ ),output_tokens,total_tokens: input_tokens.zip(output_tokens).map(|(input, output)| {input@@ -1289,7 +1294,7 @@ fn encode_anthropic_usage(usage: &Usage) -> Value {if let Some(thinking_tokens) = usage.reasoning_tokens {value["output_tokens_details"] = json!({"thinking_tokens": thinking_tokens});}- value+ crate::codecs::common::encode_usage_metadata(value, usage)}// Maps Anthropic stop reasons to normalized stop reasons.diff --git a/crates/switchyard-translation/src/codecs/anthropic/stream.rs b/crates/switchyard-translation/src/codecs/anthropic/stream.rsindex 908941a89..f901d875e 100644--- a/crates/switchyard-translation/src/codecs/anthropic/stream.rs+++ b/crates/switchyard-translation/src/codecs/anthropic/stream.rs@@ -628,6 +628,20 @@ fn capture_anthropic_usage(state: &mut StreamTranslationState, usage: &Value) {if let Some(value) = usage.get("input_tokens").and_then(Value::as_u64) {state.usage.input_tokens = Some(value);}+ if let Some(cost) = crate::codecs::common::provider_cost(usage) {+ state.usage.provider_cost = Some(cost);+ }+ if let Some(tokens) = usage+ .get("cache_creation")+ .and_then(|details| details.get("ephemeral_1h_input_tokens"))+ .and_then(Value::as_u64)+ {+ state+ .usage+ .cache+ .get_or_insert_with(Default::default)+ .cache_creation_1h_input_tokens = Some(tokens);+ }if let Some(value) = usage.get("output_tokens").and_then(Value::as_u64) {state.usage.output_tokens = Some(value);}@@ -676,7 +690,7 @@ fn anthropic_stream_usage(state: &StreamTranslationState) -> Value {json!({"thinking_tokens": value}),);}- Value::Object(usage)+ crate::codecs::common::encode_usage_metadata(Value::Object(usage), &state.usage)}// Converts any upstream message ID into an Anthropic-looking message ID.diff --git a/crates/switchyard-translation/src/codecs/common.rs b/crates/switchyard-translation/src/codecs/common.rsindex ddd795f80..303c90321 100644--- a/crates/switchyard-translation/src/codecs/common.rs+++ b/crates/switchyard-translation/src/codecs/common.rs@@ -231,6 +231,51 @@ pub(crate) fn provider_extensions(extensions}+/// Invalid charges remain unknown so accounting can use configured token rates.+pub(crate) fn provider_cost(value: &Map<String, Value>) -> Option<f64> {+ value+ .get("cost")+ .and_then(Value::as_f64)+ .filter(|cost| cost.is_finite() && *cost >= 0.0)+}++/// Cache duration must survive decoding because one-hour writes can have a different rate.+pub(crate) fn cache_usage_details(+ value: &Map<String, Value>,+ cached: Option<u64>,+ created: Option<u64>,+) -> Option<Box<crate::llm::InputCacheUsage>> {+ let one_hour = value+ .get("cache_creation")+ .and_then(|details| details.get("ephemeral_1h_input_tokens"))+ .and_then(Value::as_u64);+ let mut cache = crate::llm::Usage::cache_details(cached, created);+ if let Some(tokens) = one_hour {+ cache+ .get_or_insert_with(Default::default)+ .cache_creation_1h_input_tokens = Some(tokens);+ }+ cache+}++/// These fields preserve provider charges and cache duration across response formats.+pub(crate) fn encode_usage_metadata(mut value: Value, usage: &crate::llm::Usage) -> Value {+ if let Some(cost) = usage+ .provider_cost+ .filter(|cost| cost.is_finite() && *cost >= 0.0)+ {+ value["cost"] = serde_json::json!(cost);+ }+ if let Some(tokens) = usage+ .cache+ .as_ref()+ .and_then(|cache| cache.cache_creation_1h_input_tokens)+ {+ value["cache_creation"] = serde_json::json!({"ephemeral_1h_input_tokens": tokens});+ }+ value+}+#[cfg(test)]mod tests {use super::*;diff --git a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rsindex 907518fe9..0dc064162 100644--- a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs+++ b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs@@ -1248,6 +1248,7 @@ pub(crate) fn decode_openai_usage(value: Option<&Value>) -> Usage {}).and_then(Value::as_u64);Usage {+ provider_cost: crate::codecs::common::provider_cost(value),input_tokens: value.get("prompt_tokens").and_then(Value::as_u64)@@ -1256,7 +1257,11 @@ pub(crate) fn decode_openai_usage(value: Option<&Value>) -> Usage {.saturating_sub(cached_input_tokens.unwrap_or(0)).saturating_sub(cache_creation_input_tokens.unwrap_or(0))}),- cache: Usage::cache_details(cached_input_tokens, cache_creation_input_tokens),+ cache: crate::codecs::common::cache_usage_details(+ value,+ cached_input_tokens,+ cache_creation_input_tokens,+ ),output_tokens: value.get("completion_tokens").and_then(Value::as_u64),total_tokens: value.get("total_tokens").and_then(Value::as_u64),reasoning_tokens: value@@ -1295,7 +1300,7 @@ pub(crate) fn encode_openai_usage(usage: &Usage) -> Value {"reasoning_tokens": reasoning_tokens,});}- value+ crate::codecs::common::encode_usage_metadata(value, usage)}/// Maps OpenAI finish reasons to normalized stop reasons.diff --git a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs b/crates/switchyard-translation/src/codecs/openai_chat/stream.rsindex c86749728..02cec4b6e 100644--- a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs+++ b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs@@ -5,6 +5,7 @@use serde_json::{Map, Value, json};+use super::buffered::{decode_openai_usage, encode_openai_usage};use crate::LlmResponseChunk;use crate::codecs::common::{first_nonempty_string, reasoning_text_from_details};use crate::codecs::stream::{@@ -12,7 +13,6 @@ use crate::codecs::stream::{target_model_or_source_model,};use crate::format::{FormatId, WireFormat};-use crate::llm::Usage;/// Stream codec for OpenAI Chat Completions chunks.pub struct OpenAiChatStreamCodec;@@ -84,8 +84,8 @@ fn decode_openai_chat_stream(});}- if let Some(usage) = object.get("usage").and_then(Value::as_object) {- let usage = openai_usage(usage);+ if let Some(usage) = object.get("usage").filter(|usage| usage.is_object()) {+ let usage = decode_openai_usage(Some(usage));state.usage = usage.clone();state.saw_backend_usage = true;out.push(LlmResponseChunk::Usage(usage));@@ -324,7 +324,9 @@ fn encode_openai_chat_stream(state,json!({}),Some(openai_finish_reason(reason.as_deref())),- state.saw_backend_usage.then(|| openai_usage_value(state)),+ state+ .saw_backend_usage+ .then(|| encode_openai_usage(&state.usage)),)]}LlmResponseChunk::DecodeError { message } | LlmResponseChunk::StreamError { message } => {@@ -346,48 +348,12 @@ fn finish_openai_chat_stream(state: &mut StreamTranslationState) -> Vec<Value> {state,json!({}),Some(openai_finish_reason(state.stop_reason.as_deref())),- state.saw_backend_usage.then(|| openai_usage_value(state)),+ state+ .saw_backend_usage+ .then(|| encode_openai_usage(&state.usage)),)]}-// Normalizes OpenAI token usage fields.-fn openai_usage(usage: &Map<String, Value>) -> Usage {- let cached_input_tokens = usage- .get("prompt_tokens_details")- .and_then(|details| details.get("cached_tokens"))- .and_then(Value::as_u64);- let cache_creation_input_tokens = usage- .get("prompt_tokens_details")- .and_then(|details| {- details- .get("cache_write_tokens")- .or_else(|| details.get("cache_creation_tokens"))- })- .and_then(Value::as_u64);- Usage {- input_tokens: usage- .get("prompt_tokens")- .and_then(Value::as_u64)- .map(|tokens| {- tokens- .saturating_sub(cached_input_tokens.unwrap_or(0))- .saturating_sub(cache_creation_input_tokens.unwrap_or(0))- }),- cache: Usage::cache_details(cached_input_tokens, cache_creation_input_tokens),- output_tokens: usage.get("completion_tokens").and_then(Value::as_u64),- total_tokens: usage.get("total_tokens").and_then(Value::as_u64),- reasoning_tokens: usage- .get("completion_tokens_details")- .and_then(|details| details.get("reasoning_tokens"))- .or_else(|| {- usage- .get("output_tokens_details")- .and_then(|details| details.get("reasoning_tokens"))- })- .and_then(Value::as_u64),- }-}-// Builds a single OpenAI Chat stream chunk payload.fn openai_stream_chunk(state: &StreamTranslationState,@@ -414,7 +380,12 @@ fn openai_stream_chunk(// Builds the usage-only chunk OpenAI emits after the terminal choices chunk.fn openai_usage_chunk(state: &StreamTranslationState) -> Value {- let mut payload = openai_stream_chunk(state, json!({}), None, Some(openai_usage_value(state)));+ let mut payload = openai_stream_chunk(+ state,+ json!({}),+ None,+ Some(encode_openai_usage(&state.usage)),+ );payload["choices"] = json!([]);payload}@@ -449,34 +420,6 @@ fn openai_tool_call_chunk()}-// Builds OpenAI usage payloads from normalized and provider-extra state.-fn openai_usage_value(state: &StreamTranslationState) -> Value {- let cache_creation_tokens = state.usage.cache_creation_input_tokens().unwrap_or(0);- let cache_read_tokens = state.usage.cached_input_tokens().unwrap_or(0);- let prompt_tokens =- state.usage.input_tokens.unwrap_or(0) + cache_creation_tokens + cache_read_tokens;- let completion_tokens = state.usage.output_tokens.unwrap_or(0);- let mut usage = json!({- "prompt_tokens": prompt_tokens,- "completion_tokens": completion_tokens,- "total_tokens": state.usage.total_tokens.unwrap_or(prompt_tokens + completion_tokens),- });- if let Some(reasoning_tokens) = state.usage.reasoning_tokens {- usage["completion_tokens_details"] = json!({- "reasoning_tokens": reasoning_tokens,- });- }- if state.usage.cache_creation_input_tokens().is_some()- || state.usage.cached_input_tokens().is_some()- {- usage["prompt_tokens_details"] = json!({- "cached_tokens": cache_read_tokens,- "cache_creation_tokens": cache_creation_tokens,- });- }- usage-}-// Converts any upstream message ID into an OpenAI-looking stream ID.fn openai_stream_id(state: &StreamTranslationState) -> String {let Some(id) = state.message_id.as_deref() else {diff --git a/crates/switchyard-translation/src/codecs/responses/buffered.rs b/crates/switchyard-translation/src/codecs/responses/buffered.rsindex 43fab2806..294e59740 100644--- a/crates/switchyard-translation/src/codecs/responses/buffered.rs+++ b/crates/switchyard-translation/src/codecs/responses/buffered.rs@@ -2100,8 +2100,13 @@ fn decode_responses_usage(value: Option<&Value>) -> Usage {.or_else(|| value.get("completion_tokens")).and_then(Value::as_u64);Usage {+ provider_cost: crate::codecs::common::provider_cost(value),input_tokens,- cache: Usage::cache_details(cached_input_tokens, cache_creation_input_tokens),+ cache: crate::codecs::common::cache_usage_details(+ value,+ cached_input_tokens,+ cache_creation_input_tokens,+ ),output_tokens,total_tokens: value.get("total_tokens")@@ -2124,7 +2129,7 @@ fn decode_responses_usage(value: Option<&Value>) -> Usage {}// Encodes normalized usage into Responses usage JSON.-fn encode_responses_usage(usage: &Usage) -> Value {+pub(super) fn encode_responses_usage(usage: &Usage) -> Value {let input_tokens = usage.input_tokens.unwrap_or(0)+ usage.cached_input_tokens().unwrap_or(0)+ usage.cache_creation_input_tokens().unwrap_or(0);@@ -2132,19 +2137,22 @@ fn encode_responses_usage(usage: &Usage) -> Value {// upstream that reports no cache or reasoning breakdown means zero, not "unknown"; omitting// them instead yields a payload that fails to deserialize into the OpenAI SDK's// ResponseUsage, which types both as non-optional.- json!({- "input_tokens": input_tokens,- "output_tokens": usage.output_tokens.unwrap_or(0),- "total_tokens": usage- .total_tokens- .or_else(|| Some(input_tokens + usage.output_tokens.unwrap_or(0)))- .unwrap_or(0),- "input_tokens_details": {- "cached_tokens": usage.cached_input_tokens().unwrap_or(0),- "cache_write_tokens": usage.cache_creation_input_tokens().unwrap_or(0),- },- "output_tokens_details": {"reasoning_tokens": usage.reasoning_tokens.unwrap_or(0)},- })+ crate::codecs::common::encode_usage_metadata(+ json!({+ "input_tokens": input_tokens,+ "output_tokens": usage.output_tokens.unwrap_or(0),+ "total_tokens": usage+ .total_tokens+ .or_else(|| Some(input_tokens + usage.output_tokens.unwrap_or(0)))+ .unwrap_or(0),+ "input_tokens_details": {+ "cached_tokens": usage.cached_input_tokens().unwrap_or(0),+ "cache_write_tokens": usage.cache_creation_input_tokens().unwrap_or(0),+ },+ "output_tokens_details": {"reasoning_tokens": usage.reasoning_tokens.unwrap_or(0)},+ }),+ usage,+ )}/// Re-emits captured request extensions that the Responses format accepts.diff --git a/crates/switchyard-translation/src/codecs/responses/stream.rs b/crates/switchyard-translation/src/codecs/responses/stream.rsindex 35f3a1411..b726bea71 100644--- a/crates/switchyard-translation/src/codecs/responses/stream.rs+++ b/crates/switchyard-translation/src/codecs/responses/stream.rs@@ -17,6 +17,8 @@ use crate::codecs::stream::{use crate::format::{FormatId, WireFormat};use crate::llm::Usage;+use super::buffered::encode_responses_usage;+/// Stream codec for OpenAI Responses API events.pub struct OpenAiResponsesStreamCodec;@@ -850,7 +852,7 @@ fn responses_stream_response(tool_choice: "auto",tools: Vec::new(),top_p: None,- usage: responses_usage_value(&state.usage),+ usage: encode_responses_usage(&state.usage),}}@@ -1163,8 +1165,13 @@ fn responses_usage(usage: &serde_json::Map<String, Value>) -> Usage {});let output_tokens = usage.get("output_tokens").and_then(Value::as_u64);Usage {+ provider_cost: crate::codecs::common::provider_cost(usage),input_tokens,- cache: Usage::cache_details(cached_input_tokens, cache_creation_input_tokens),+ cache: crate::codecs::common::cache_usage_details(+ usage,+ cached_input_tokens,+ cache_creation_input_tokens,+ ),output_tokens,total_tokens: usage.get("total_tokens")@@ -1182,27 +1189,6 @@ fn responses_usage(usage: &serde_json::Map<String, Value>) -> Usage {}}-// Builds OpenAI Responses usage payloads from normalized usage.-fn responses_usage_value(usage: &Usage) -> Value {- let input_tokens = usage.input_tokens.unwrap_or(0)- + usage.cached_input_tokens().unwrap_or(0)- + usage.cache_creation_input_tokens().unwrap_or(0);- // Both detail objects are always present, for the same reason as the buffered encoder: the- // Responses schema types them as required, so a missing breakdown serializes as zero.- json!({- "input_tokens": input_tokens,- "output_tokens": usage.output_tokens.unwrap_or(0),- "total_tokens": usage.total_tokens.unwrap_or_else(|| {- input_tokens + usage.output_tokens.unwrap_or(0)- }),- "input_tokens_details": {- "cached_tokens": usage.cached_input_tokens().unwrap_or(0),- "cache_write_tokens": usage.cache_creation_input_tokens().unwrap_or(0),- },- "output_tokens_details": {"reasoning_tokens": usage.reasoning_tokens.unwrap_or(0)},- })-}-// Longest response-id discriminator embedded verbatim in a synthesized item id. OpenAI rejects// item ids over 64 characters, and some upstreams issue response ids several hundred characters// long, so anything longer is replaced by a fixed-width digest.diff --git a/docs/cli_reference.md b/docs/cli_reference.mdindex e5da32455..c9374fbb9 100644--- a/docs/cli_reference.md+++ b/docs/cli_reference.md@@ -20,6 +20,7 @@ switchyard-server --config <deployment.toml> [options]| `--shutdown-timeout SHUTDOWN_TIMEOUT` | `30s` | Maximum time active requests may drain during shutdown. || `--dry-run` | Off | Validate the deployment without binding a socket. || `--routing-log-file PATH` | None | Append durable per-request routing records to this JSONL file. |+| `--pricing-file PATH` | None | Estimate unreported call costs from this JSON file of USD rates per million tokens; requires `--routing-log-file`. || `--tls-cert PATH` | None | PEM certificate path; requires `--tls-key`. || `--tls-key PATH` | None | PEM private-key path; requires `--tls-cert`. || `-h, --help` | — | Print command help. |diff --git a/docs/integrations/pi.md b/docs/integrations/pi.mdindex 75a2853b3..1c71f9aa3 100644--- a/docs/integrations/pi.md+++ b/docs/integrations/pi.md@@ -155,15 +155,18 @@ second shows `cached_tokens` close to `prompt_tokens`. If the second record show#### Estimate the cost-The routing log records token counts, not prices. To estimate what a request cost,-multiply each count in its record by the matching price and add the results:+The server matches each token count in the routing log to a rate in the pricing file:-| Tokens in the record | Price |+| Tokens in the record | Pricing-file key ||---|---|-| `prompt_tokens - cached_tokens - cache_creation_tokens` | Input |-| `cached_tokens` | Cache read |-| `cache_creation_tokens` | Cache write |-| `completion_tokens` | Output |+| `prompt_tokens - cached_tokens - cache_creation_tokens` | `input` |+| `cached_tokens` | `cache_read` |+| `cache_creation_tokens - cache_creation_1h_tokens` | `cache_write` |+| `cache_creation_1h_tokens`, when reported | `cache_write_1h` |+| `completion_tokens` | `output` |++Treat a missing `cache_creation_1h_tokens` count as zero. That count is part of+`cache_creation_tokens`, so it must not be charged twice.Write the prices to `prices.json` in USD per million tokens, and key each entry by therecord's `model` value. This example uses Anthropic's list prices for Claude Sonnet 5 on@@ -175,25 +178,30 @@ record's `model` value. This example uses Anthropic's list prices for Claude Son}```+For your own models, copy the entry for each model and replace the key with its exact+`model` value from the routing log. Use the served model ID rather than the routing alias,+and replace the example's numbers with your provider's rates. List recorded model IDs with:++```bash+jq -r '.model' PATH | sort -u+```++Every entry requires `input`, `output`, `cache_read`, and `cache_write`. Set a rate to zero+only when that usage is free. Each rate must be a finite number from 0 to 10000 USD per+million tokens.+The example's `cache_write` price is for a 5-minute cache. A 1-hour cache costs 2 times-the input price instead of 1.25 times. The routing log does not say which cache the-gateway used. A gateway may also charge its own prices, so the result is an estimate, not-the gateway's bill. This command prints one estimated cost per record, or a warning for a-model that has no entry in `prices.json`:+the input price instead of 1.25 times; set `cache_write_1h` to `4.00` for that example.+Include that rate to estimate reported one-hour cache writes; without it, the estimate+remains unknown.++Start the server with `--pricing-file prices.json` as well as `--routing-log-file PATH`.+The server records the gateway's reported charge when available, even without rates.+Otherwise it uses the pricing file to estimate the cost. If usage is missing or the model+has no rates, the estimate is unknown. Read each request's cost with:```bash-jq -r --slurpfile prices prices.json '- . as $r- | ($prices[0][$r.model // ""]) as $p- | if $p == null then- "warning: no price for model \($r.model); add it to prices.json"- else- ((($r.prompt_tokens // 0) - ($r.cached_tokens // 0) - ($r.cache_creation_tokens // 0)) * $p.input- + ($r.cached_tokens // 0) * $p.cache_read- + ($r.cache_creation_tokens // 0) * $p.cache_write- + ($r.completion_tokens // 0) * $p.output) / 1000000- | "\($r.route_id) \($r.model) estimated $\(. * 1000000 | round / 1000000)"- end' PATH+jq -c '{route_id, model, cost}' PATH```### Thinkingdiff --git a/scripts/macos/install.sh b/scripts/macos/install.shindex ae7b3a9dd..f3cff8135 100755--- a/scripts/macos/install.sh+++ b/scripts/macos/install.sh@@ -55,8 +55,8 @@ step "Writing server config"write_once "$SY_HOME/composite.toml" < "$REPO_ROOT/scripts/config/composite.toml"step "Writing menu bar settings"-# Rates are list prices per million tokens and are only used to estimate-# savings. Edit them to match what you actually pay.+# The server and menubar share rates in USD per million tokens.+# Edit prices.json to match what you actually pay.TOML_SY_HOME="$(toml_escape_basic_string "$SY_HOME")"write_once "$SY_HOME/menubar.toml" <<EOFserver_url = "http://127.0.0.1:$SY_PORT"@@ -70,52 +70,26 @@ refresh_seconds = 30# Switchyard's own classifier calls counted against it.baseline_model = "gpt-5.6-sol"-[prices."gpt-5.6-sol"]-input_per_mtok = 1.25-cached_input_per_mtok = 0.125-output_per_mtok = 10.0--[prices."gpt-5.6-luna"]-input_per_mtok = 0.25-cached_input_per_mtok = 0.025-output_per_mtok = 2.0--[prices."gpt-5.6-terra"]-input_per_mtok = 0.05-cached_input_per_mtok = 0.005-output_per_mtok = 0.4--[prices."gpt-6-luna"]-input_per_mtok = 0.10-cached_input_per_mtok = 0.01-output_per_mtok = 0.50--[prices."gpt-6-sol"]-input_per_mtok = 2.00-cached_input_per_mtok = 0.20-output_per_mtok = 10.00--[prices."gpt-6.1-sol"]-input_per_mtok = 2.00-cached_input_per_mtok = 0.10-output_per_mtok = 10.00--# With a ChatGPT login, Codex sends "Approve for me" reviews to-# codex-auto-review even without Switchyard, so this price copies the-# baseline_model rates. Reviews then add the same amount to the actual cost and-# the baseline. The dollars saved stay the same, but the percentage drops a-# little. Update this price if you change baseline_model.-[prices."codex-auto-review"]-input_per_mtok = 1.25-cached_input_per_mtok = 0.125-output_per_mtok = 10.0+pricing_file = "$TOML_SY_HOME/prices.json"+EOF++write_once "$SY_HOME/prices.json" <<'EOF'+{+ "gpt-5.6-sol": {"input": 1.25, "cache_read": 0.125, "cache_write": 1.25, "output": 10.0},+ "gpt-5.6-luna": {"input": 0.25, "cache_read": 0.025, "cache_write": 0.25, "output": 2.0},+ "gpt-5.6-terra": {"input": 0.05, "cache_read": 0.005, "cache_write": 0.05, "output": 0.4},+ "gpt-6-luna": {"input": 0.1, "cache_read": 0.01, "cache_write": 0.1, "output": 0.5},+ "gpt-6-sol": {"input": 2.0, "cache_read": 0.2, "cache_write": 2.0, "output": 10.0},+ "gpt-6.1-sol": {"input": 2.0, "cache_read": 0.1, "cache_write": 2.0, "output": 10.0},+ "codex-auto-review": {"input": 1.25, "cache_read": 0.125, "cache_write": 1.25, "output": 10.0}+}EOFstep "Validating the server config"if (( DRY_RUN )); then- say " would run: $SY_HOME/bin/switchyard-server --config $SY_HOME/composite.toml --dry-run"+ say " would run: $SY_HOME/bin/switchyard-server --config $SY_HOME/composite.toml --routing-log-file $SY_HOME/routing.jsonl --pricing-file $SY_HOME/prices.json --dry-run"else- "$SY_HOME/bin/switchyard-server" --config "$SY_HOME/composite.toml" --dry-run+ "$SY_HOME/bin/switchyard-server" --config "$SY_HOME/composite.toml" --routing-log-file "$SY_HOME/routing.jsonl" --pricing-file "$SY_HOME/prices.json" --dry-runfistep "Writing LaunchAgents"@@ -138,6 +112,8 @@ write_always "$LAUNCH_AGENTS/$SERVER_LABEL.plist" <<EOF<string>$SY_PORT</string><string>--routing-log-file</string><string>$XML_SY_HOME/routing.jsonl</string>+ <string>--pricing-file</string>+ <string>$XML_SY_HOME/prices.json</string></array><key>RunAtLoad</key><true/>