Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
19.3 KB
Raw Blame
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

//! Durable per-request routing records and session snapshots.

use std::borrow::Cow;
use std::collections::BTreeMap;
use std::fs;
use std::io::{BufRead, BufReader, Write};
use std::path::{Path, PathBuf};
use std::time::SystemTime;

use humantime::format_rfc3339_millis;
use libsy::OutcomeMetadata;
use serde::{Deserialize, Serialize};
use switchyard_protocol::{Metadata, ModelId, Usage};

use crate::usage_metrics::token_usage;
use crate::{ServerError, ServerResult};
use switchyard_costs::{CostTotals, Pricing, RecordedCost};

const LEGACY_SESSION_ID_HEADER: &str = "proxy_x_session_id";
const ORIGIN_HEADER: &str = "x-switchyard-origin";
const TASK_HEADER: &str = "x-switchyard-intake-task";
const TRIAL_ID_HEADER: &str = "x-switchyard-trial-id";

/// Append-only writer for one routing JSONL file.
pub(crate) struct RoutingLog {
    file: fs::File,
    pub(crate) pricing: Pricing,
}

impl RoutingLog {
    pub(crate) fn new(path: impl Into<PathBuf>) -> ServerResult<Self> {
        let path = path.into();
        if let Some(parent) = path
            .parent()
            .filter(|parent| !parent.as_os_str().is_empty())
        {
            fs::create_dir_all(parent).map_err(|error| routing_log_error(&path, error))?;
        }
        let file = fs::OpenOptions::new()
            .create(true)
            .append(true)
            .open(&path)
            .map_err(|error| routing_log_error(&path, error))?;
        Ok(Self {
            file,
            pricing: Pricing::bundled().map_err(ServerError::new)?,
        })
    }

    /// Writes one record for a finished call as a single JSON line.
    pub(crate) fn append(
        &mut self,
        context: RoutingLogContext,
        target: &str,
        model: &str,
        tier: Option<&str>,
        usage: &Usage,
    ) -> std::io::Result<()> {
        let cost = self.pricing.cost(model, usage);
        let cache_creation_1h_tokens = usage
            .cache
            .as_ref()
            .and_then(|cache| cache.cache_creation_1h_input_tokens)
            .unwrap_or(0);
        let usage = token_usage(usage);
        let record = RoutingRecord {
            outcome: None,
            cost,
            ts: format_rfc3339_millis(SystemTime::now()).to_string().into(),
            route_id: context.route_id.into(),
            algorithm: context.algorithm.into(),
            origin: context.origin.map(Cow::Owned),
            task: context.task.map(Cow::Owned),
            trial_id: context.trial_id.map(Cow::Owned),
            session_id: context.session_id.map(Cow::Owned),
            turn_id: context.turn_id.map(Cow::Owned),
            agent_id: context.agent.agent_id.map(Cow::Owned),
            parent_agent_id: context.agent.parent_agent_id.map(Cow::Owned),
            delegated: context.agent.delegated,
            agent_kind: context.agent.kind.map(Cow::Owned),
            agent_role: context.agent.role.map(Cow::Owned),
            subagent_identity_unsupported: context.agent.identity_unsupported,
            model: model.into(),
            target: target.into(),
            reasoning_effort: context.reasoning_effort.map(Cow::Owned),
            tier: tier.unwrap_or("").into(),
            prompt_tokens: usage.prompt_tokens,
            cached_tokens: usage.cached_tokens,
            cache_creation_tokens: usage.cache_creation_tokens,
            cache_creation_1h_tokens,
            completion_tokens: usage.completion_tokens,
            reasoning_tokens: usage.reasoning_tokens,
            total_tokens: usage.prompt_tokens.saturating_add(usage.completion_tokens),
        };
        self.append_record(&record)
    }

    pub(crate) fn append_outcome(
        &mut self,
        context: &RoutingLogContext,
        metadata: OutcomeMetadata,
    ) -> std::io::Result<()> {
        self.append_record(&serde_json::json!({
            "session_id": context.session_id,
            "outcome": LoggedOutcome {
                ts: format_rfc3339_millis(SystemTime::now()).to_string(),
                route_id: context.route_id.clone(),
                turn_id: context.turn_id.clone(),
                agent_id: context.agent.agent_id.clone(),
                parent_agent_id: context.agent.parent_agent_id.clone(),
                metadata,
            },
        }))
    }

    fn append_record(&mut self, record: &impl Serialize) -> std::io::Result<()> {
        let mut line = serde_json::to_vec(record).map_err(std::io::Error::other)?;
        line.push(b'\n');

        self.file.write_all(&line)
    }
}

/// Reads complete records without synchronizing with the writer.
pub(crate) fn snapshot(
    path: &Path,
    session_id: &str,
) -> std::io::Result<Option<SessionStatsSnapshot>> {
    let mut reader = BufReader::with_capacity(64 * 1024, fs::File::open(path)?);
    let mut line = Vec::new();
    let mut snapshot = SessionStatsSnapshot {
        session_id: session_id.to_string(),
        ..Default::default()
    };
    loop {
        line.clear();
        if reader.read_until(b'\n', &mut line)? == 0 {
            break;
        }
        if !line.ends_with(b"\n") {
            break;
        }
        let Ok(record) = serde_json::from_slice::<RoutingRecord>(&line) else {
            continue;
        };
        snapshot.add_record(record, session_id);
    }
    snapshot.sum_totals();
    Ok((snapshot.total_calls > 0 || !snapshot.outcomes.is_empty()).then_some(snapshot))
}

/// Which agent made a call. A child agent can share its parent's session id, so
/// the session alone cannot tell delegated work apart from the parent's turns.
/// Every field is copied from the request metadata, never inferred.
#[derive(Clone, Default)]
pub(crate) struct AgentIdentity {
    agent_id: Option<String>,
    parent_agent_id: Option<String>,
    /// [`Metadata::is_subagent_work`], the signal that sends a call to the
    /// sub-agent route, rather than the weaker `is_subagent` hint.
    delegated: bool,
    kind: Option<String>,
    role: Option<String>,
    /// The harness cannot identify its children, so a reader should not take
    /// a missing `delegated` flag to mean that no work was delegated.
    identity_unsupported: bool,
}

impl AgentIdentity {
    /// Copies the agent fields from the request metadata.
    fn from_metadata(metadata: &Metadata) -> Self {
        Self {
            agent_id: metadata.agent_id.clone(),
            parent_agent_id: metadata.parent_agent_id.clone(),
            delegated: metadata.is_subagent_work(),
            kind: metadata.agent_kind.clone(),
            role: metadata.agent_role.clone(),
            identity_unsupported: metadata.subagent_identity_unsupported,
        }
    }
}

/// Holds request metadata until route resolution attaches the durable route identity.
#[derive(Clone)]
pub(crate) struct RoutingLogContext {
    route_id: String,
    algorithm: String,
    origin: Option<String>,
    task: Option<String>,
    trial_id: Option<String>,
    session_id: Option<String>,
    turn_id: Option<String>,
    agent: AgentIdentity,
    reasoning_effort: Option<String>,
}

impl RoutingLogContext {
    /// Captures the normalized session ID, with the legacy log-only header as a fallback.
    pub(crate) fn from_metadata(metadata: &Metadata) -> Self {
        let headers = metadata.http_headers.as_ref();
        Self {
            route_id: String::new(),
            algorithm: String::new(),
            origin: headers
                .and_then(|headers| nonempty_header(headers, ORIGIN_HEADER))
                .map(str::to_string),
            task: headers
                .and_then(|headers| nonempty_header(headers, TASK_HEADER))
                .map(str::to_string),
            trial_id: headers
                .and_then(|headers| nonempty_header(headers, TRIAL_ID_HEADER))
                .map(str::to_string),
            turn_id: metadata.turn_id.clone(),
            session_id: metadata.session_id.clone().or_else(|| {
                headers
                    .and_then(|headers| nonempty_header(headers, LEGACY_SESSION_ID_HEADER))
                    .map(str::to_string)
            }),
            agent: AgentIdentity::from_metadata(metadata),
            reasoning_effort: None,
        }
    }

    /// Captures configured effort for this particular target's call.
    pub(crate) fn with_reasoning_effort(mut self, effort: Option<&str>) -> Self {
        self.reasoning_effort = effort.map(str::to_owned);
        self
    }

    /// Attaches the resolved route identity shared by every log entry for this request.
    pub(crate) fn with_route(mut self, route_id: impl Into<String>, algorithm: &str) -> Self {
        self.route_id = route_id.into();
        self.algorithm = algorithm.to_string();
        self
    }
}

/// One appended routing record, and the read schema [`snapshot`] parses back,
/// so the written and expected shapes cannot drift apart. Missing fields
/// default so a record from an older schema still contributes what it has.
#[derive(Default, Deserialize, Serialize)]
#[serde(default)]
struct RoutingRecord<'a> {
    #[serde(skip_serializing_if = "Option::is_none")]
    outcome: Option<LoggedOutcome>,
    #[serde(default)]
    cost: Option<RecordedCost>,
    ts: Cow<'a, str>,
    route_id: Cow<'a, str>,
    algorithm: Cow<'a, str>,
    #[serde(borrow)]
    origin: Option<Cow<'a, str>>,
    #[serde(borrow)]
    task: Option<Cow<'a, str>>,
    #[serde(borrow)]
    trial_id: Option<Cow<'a, str>>,
    #[serde(borrow)]
    session_id: Option<Cow<'a, str>>,
    #[serde(borrow)]
    turn_id: Option<Cow<'a, str>>,
    // Most calls carry no agent identity, so these fields are written only when
    // set. Records without them still parse through `serde(default)`.
    #[serde(borrow, skip_serializing_if = "Option::is_none")]
    agent_id: Option<Cow<'a, str>>,
    #[serde(borrow, skip_serializing_if = "Option::is_none")]
    parent_agent_id: Option<Cow<'a, str>>,
    #[serde(skip_serializing_if = "is_false")]
    delegated: bool,
    #[serde(borrow, skip_serializing_if = "Option::is_none")]
    agent_kind: Option<Cow<'a, str>>,
    #[serde(borrow, skip_serializing_if = "Option::is_none")]
    agent_role: Option<Cow<'a, str>>,
    #[serde(skip_serializing_if = "is_false")]
    subagent_identity_unsupported: bool,
    /// Provider model that served the call, which is what cost tools price.
    model: Cow<'a, str>,
    /// Configured target that served the call. Targets may share a model.
    #[serde(default, borrow)]
    target: Cow<'a, str>,
    #[serde(borrow, skip_serializing_if = "Option::is_none")]
    reasoning_effort: Option<Cow<'a, str>>,
    tier: Cow<'a, str>,
    prompt_tokens: u64,
    cached_tokens: u64,
    cache_creation_tokens: u64,
    /// The part of `cache_creation_tokens` written at the one-hour rate, when reported.
    #[serde(skip_serializing_if = "is_zero")]
    cache_creation_1h_tokens: u64,
    completion_tokens: u64,
    reasoning_tokens: u64,
    total_tokens: u64,
}

fn is_zero(value: &u64) -> bool {
    *value == 0
}

/// Session totals returned by the routing stats endpoint.
#[derive(Default, Serialize)]
pub(crate) struct SessionStatsSnapshot {
    session_id: String,
    cost: CostTotals,
    answer_cost: CostTotals,
    routing_cost: CostTotals,
    total_calls: u64,
    total_prompt_tokens: u64,
    total_cached_tokens: u64,
    total_cache_creation_tokens: u64,
    total_completion_tokens: u64,
    models: BTreeMap<ModelId, SessionModelStats>,
    targets: BTreeMap<ModelId, SessionTargetStats>,
    #[serde(skip_serializing_if = "Vec::is_empty")]
    outcomes: Vec<LoggedOutcome>,
}

#[derive(Deserialize, Serialize)]
struct LoggedOutcome {
    ts: String,
    route_id: String,
    turn_id: Option<String>,
    #[serde(skip_serializing_if = "Option::is_none")]
    agent_id: Option<String>,
    #[serde(skip_serializing_if = "Option::is_none")]
    parent_agent_id: Option<String>,
    #[serde(flatten)]
    metadata: OutcomeMetadata,
}

#[derive(Default, Serialize)]
struct SessionModelStats {
    cost: CostTotals,
    calls: u64,
    prompt_tokens: u64,
    cached_tokens: u64,
    cache_creation_tokens: u64,
    completion_tokens: u64,
}

#[derive(Serialize)]
struct SessionTargetStats {
    model: String,
    reasoning_effort: Option<String>,
    #[serde(flatten)]
    stats: SessionModelStats,
}

impl SessionModelStats {
    fn add_record(&mut self, record: &RoutingRecord<'_>) {
        self.cost.add(record.cost);
        self.calls = self.calls.saturating_add(1);
        self.prompt_tokens = self.prompt_tokens.saturating_add(record.prompt_tokens);
        self.cached_tokens = self.cached_tokens.saturating_add(record.cached_tokens);
        self.cache_creation_tokens = self
            .cache_creation_tokens
            .saturating_add(record.cache_creation_tokens);
        self.completion_tokens = self
            .completion_tokens
            .saturating_add(record.completion_tokens);
    }
}

impl SessionStatsSnapshot {
    fn add_record(&mut self, mut record: RoutingRecord<'_>, session_id: &str) {
        if record.session_id.as_deref() != Some(session_id) {
            return;
        }
        if let Some(outcome) = record.outcome.take() {
            self.outcomes.push(outcome);
            return;
        }
        let model = match record.model.as_ref() {
            "" => "unknown",
            model => model,
        };
        let stats = self.models.entry(ModelId::from(model)).or_default();
        stats.add_record(&record);
        // Older logs have no target identity; keep them only in the model aggregate.
        if !record.target.is_empty() {
            self.targets
                .entry(ModelId::from(record.target.as_ref()))
                .or_insert_with(|| SessionTargetStats {
                    model: model.to_string(),
                    reasoning_effort: record.reasoning_effort.as_deref().map(str::to_owned),
                    stats: SessionModelStats::default(),
                })
                .stats
                .add_record(&record);
        }
        self.cost.add(record.cost);
        if record.tier == "classifier" {
            self.routing_cost.add(record.cost);
        } else {
            self.answer_cost.add(record.cost);
        }
    }

    /// Session totals are exactly the sum of the per-model stats, so they are
    /// derived once rather than accumulated alongside them.
    fn sum_totals(&mut self) {
        for stats in self.models.values() {
            self.total_calls = self.total_calls.saturating_add(stats.calls);
            self.total_prompt_tokens = self.total_prompt_tokens.saturating_add(stats.prompt_tokens);
            self.total_cached_tokens = self.total_cached_tokens.saturating_add(stats.cached_tokens);
            self.total_cache_creation_tokens = self
                .total_cache_creation_tokens
                .saturating_add(stats.cache_creation_tokens);
            self.total_completion_tokens = self
                .total_completion_tokens
                .saturating_add(stats.completion_tokens);
        }
    }
}

/// Lets serde leave out a `false` flag, so it is written only when set.
fn is_false(flag: &bool) -> bool {
    !flag
}

/// Returns a header value only when it is present, non-empty, and valid text.
fn nonempty_header<'a>(headers: &'a http::HeaderMap, name: &str) -> Option<&'a str> {
    headers
        .get(name)
        .filter(|value| !value.is_empty())
        .and_then(|v| v.to_str().ok())
}

fn routing_log_error(path: &Path, error: std::io::Error) -> ServerError {
    ServerError::new(format!(
        "failed to initialize routing log {}: {error}",
        path.display()
    ))
}

#[cfg(test)]
mod tests {
    use super::*;

    /// Missing, empty, and non-text origin headers remain absent even with a User-Agent.
    #[test]
    fn unusable_origin_is_not_inferred_from_user_agent() {
        for origin in [
            None,
            Some(http::HeaderValue::from_static("")),
            Some(http::HeaderValue::from_bytes(b"\xff").expect("header")),
        ] {
            let mut headers = http::HeaderMap::new();
            headers.insert(
                "user-agent",
                http::HeaderValue::from_static("codex-cli/1.0"),
            );
            if let Some(origin) = origin {
                headers.insert(ORIGIN_HEADER, origin);
            }
            let metadata = Metadata {
                http_headers: Some(headers),
                ..Default::default()
            };
            assert!(RoutingLogContext::from_metadata(&metadata).origin.is_none());
        }
        assert!(
            RoutingLogContext::from_metadata(&Metadata::default())
                .origin
                .is_none()
        );
    }

    #[test]
    fn a_delegated_call_records_its_agent_lineage() {
        let metadata = Metadata {
            session_id: Some("session-1".to_string()),
            turn_id: Some("turn-1".to_string()),
            agent_id: Some("child-7".to_string()),
            parent_agent_id: Some("thread-1".to_string()),
            is_subagent: true,
            is_delegated_work: true,
            agent_kind: Some("review".to_string()),
            agent_role: Some("reviewer".to_string()),
            ..Metadata::default()
        };
        let dir = tempfile::tempdir().expect("temp dir");
        let path = dir.path().join("routing.jsonl");
        let mut log = RoutingLog::new(&path).expect("routing log");
        log.append(
            RoutingLogContext::from_metadata(&metadata).with_route("model-router", "passthrough"),
            "worker",
            "worker-model",
            None,
            &Usage::default(),
        )
        .expect("append");
        let line = fs::read_to_string(&path).expect("read log");
        let record: serde_json::Value = serde_json::from_str(line.trim()).expect("json record");

        assert_eq!(record["session_id"], "session-1");
        assert_eq!(record["turn_id"], "turn-1");
        assert_eq!(record["agent_id"], "child-7");
        assert_eq!(record["parent_agent_id"], "thread-1");
        assert_eq!(record["delegated"], true);
        assert_eq!(record["agent_kind"], "review");
        assert_eq!(record["agent_role"], "reviewer");
        assert!(record.get("subagent_identity_unsupported").is_none());
    }

    /// Only the requested session is counted, absent fields fall back to zero
    /// and `unknown`, and an unparseable line does not abort the scan.
    #[test]
    fn snapshot_counts_only_the_requested_session() {
        let dir = tempfile::tempdir().expect("temp dir");
        let path = dir.path().join("routing.jsonl");
        fs::write(
            &path,
            concat!(
                r#"{"session_id":"a","model":"m1","fallback_reason":"unavailable","prompt_tokens":10,"completion_tokens":2}"#,
                "\n",
                r#"{"session_id":"b","model":"m1","prompt_tokens":99,"completion_tokens":99}"#,
                "\n",
                "not json\n",
                r#"{"session_id":"a","origin":"custom-agent","prompt_tokens":5}"#,
                "\n",
            ),
        )
        .expect("write log");

        let stats = snapshot(&path, "a").expect("read log").expect("session a");
        assert_eq!(stats.total_calls, 2);
        assert_eq!(stats.total_prompt_tokens, 15);
        assert_eq!(stats.total_completion_tokens, 2);
        assert_eq!(stats.models["m1"].calls, 1);
        assert!(stats.targets.is_empty());
        assert_eq!(stats.models["unknown"].prompt_tokens, 5);
        assert!(snapshot(&path, "missing").expect("read log").is_none());
    }
}