Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
15.7 KB
Raw Blame
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

//! This module totals recorded model calls from the retained routing log.

use std::collections::BTreeMap;
use std::fs::File;
use std::io::{BufRead, BufReader, Seek, SeekFrom};
use std::path::Path;

use crate::history::{Entry, parse_entry};
use chrono::{DateTime, Local, NaiveDate};
use serde::{Deserialize, Serialize};
use switchyard_costs::CostTotals;

/// CLASSIFIER_TIER matches the tier recorded for classifier and judge calls.
pub const CLASSIFIER_TIER: &str = "classifier";

#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
pub struct ModelTokens {
    pub requests: u64,
    /// This field counts input tokens that were neither read from nor written to a cache.
    pub input: u64,
    /// This field counts input tokens served from the provider cache.
    pub cached_input: u64,
    /// This field counts input tokens written to the provider cache.
    pub cache_write: u64,
    /// This field counts the cache writes that use the one-hour rate.
    pub cache_write_1h: u64,
    /// This field counts generated tokens, including reasoning.
    pub output: u64,
    /// This field adds up the costs that the server recorded for these calls.
    pub cost: CostTotals,
    /// This field counts calls recorded without token counts, which cannot be priced.
    pub uncounted: u64,
}

impl ModelTokens {
    /// This function returns the recorded token total, including cached reads.
    pub fn total(&self) -> u64 {
        self.input + self.cached_input + self.cache_write + self.output
    }

    /// This method adds totals from a disjoint set of calls.
    pub fn add(&mut self, other: &Self) {
        self.requests += other.requests;
        self.input += other.input;
        self.cached_input += other.cached_input;
        self.cache_write += other.cache_write;
        self.cache_write_1h += other.cache_write_1h;
        self.output += other.output;
        self.cost.merge(&other.cost);
        self.uncounted += other.uncounted;
    }

    /// This function returns the token counts as usage that a price table can price.
    pub fn usage(&self) -> switchyard_protocol::Usage {
        let mut cache = switchyard_protocol::Usage::cache_details(
            Some(self.cached_input),
            Some(self.cache_write),
        );
        if let Some(cache) = cache.as_mut() {
            cache.cache_creation_1h_input_tokens = Some(self.cache_write_1h);
        }
        switchyard_protocol::Usage {
            input_tokens: Some(self.input),
            output_tokens: Some(self.output),
            cache,
            ..Default::default()
        }
    }
}

/// Totals separates caller-facing calls from classifier calls for a period.
/// Request counts include caller-facing calls; token and cost totals include both groups.
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
pub struct Totals {
    /// This field groups caller-facing calls by the model that answered.
    pub routed: BTreeMap<String, ModelTokens>,
    /// This field groups classifier and judge calls by model.
    pub classifier: BTreeMap<String, ModelTokens>,
    /// Each route includes its answer calls and routing overhead.
    pub routes: BTreeMap<String, ModelTokens>,
}

impl Totals {
    /// Each call contributes to its answer or classifier totals and its route totals.
    fn add(&mut self, record: &Entry) {
        let bucket = if record.tier == CLASSIFIER_TIER {
            &mut self.classifier
        } else {
            &mut self.routed
        };
        let tokens = ModelTokens::from(record);
        bucket.entry(record.model.clone()).or_default().add(&tokens);
        self.routes
            .entry(record.route_id.clone())
            .or_default()
            .add(&tokens);
    }

    /// This method adds totals from a disjoint period without counting route totals twice.
    fn merge(&mut self, other: &Self) {
        for (own, incoming) in [
            (&mut self.routed, &other.routed),
            (&mut self.classifier, &other.classifier),
            (&mut self.routes, &other.routes),
        ] {
            for (model, tokens) in incoming {
                own.entry(model.clone()).or_default().add(tokens);
            }
        }
    }

    /// This function returns the number of caller-facing calls.
    pub fn requests(&self) -> u64 {
        self.routed.values().map(|tokens| tokens.requests).sum()
    }

    /// This function totals recorded tokens, including routing overhead.
    pub fn tokens(&self) -> u64 {
        self.routed
            .values()
            .chain(self.classifier.values())
            .map(ModelTokens::total)
            .sum()
    }

    /// This function adds up the recorded cost of every call, including routing overhead.
    pub fn cost(&self) -> CostTotals {
        let mut cost = CostTotals::default();
        for tokens in self.routed.values().chain(self.classifier.values()) {
            cost.merge(&tokens.cost);
        }
        cost
    }

    /// This function returns true when neither group contains recorded calls.
    pub fn is_empty(&self) -> bool {
        self.routed.is_empty() && self.classifier.is_empty()
    }
}

/// Usage stores totals from the retained log and seven local calendar days.
#[derive(Clone, Debug, Default, Serialize)]
pub struct Usage {
    pub today: Totals,
    pub week: Totals,
    pub all: Totals,
    pub days: BTreeMap<NaiveDate, Totals>,
}

impl From<&Entry> for ModelTokens {
    fn from(record: &Entry) -> Self {
        let mut cost = CostTotals::default();
        cost.add(record.cost);
        Self {
            requests: 1,
            // The recorded input includes cache reads and writes; output includes reasoning.
            input: record
                .prompt_tokens
                .saturating_sub(record.cached_tokens)
                .saturating_sub(record.cache_creation_tokens),
            cached_input: record.cached_tokens,
            cache_write: record.cache_creation_tokens,
            cache_write_1h: record.cache_creation_1h_tokens,
            output: record.completion_tokens,
            cost,
            uncounted: u64::from(
                record.completion_tokens == 0
                    || record.prompt_tokens == 0
                    || record
                        .prompt_tokens
                        .checked_sub(record.cached_tokens)
                        .and_then(|input| input.checked_sub(record.cache_creation_tokens))
                        .is_none()
                    || record.cache_creation_1h_tokens > record.cache_creation_tokens,
            ),
        }
    }
}

/// Reader keeps model and route totals without retaining individual calls.
/// Each read consumes only newly completed log lines.
#[derive(Default)]
pub struct Reader {
    file_id: Option<(u64, u64)>,
    offset: u64,
    days: BTreeMap<NaiveDate, Totals>,
    all: Totals,
}

impl Reader {
    /// A missing log has no usage. Replacement or truncation starts fresh.
    /// An invalid complete record returns an error because skipping it would understate usage.
    pub fn read(&mut self, path: &Path, today: NaiveDate) -> std::io::Result<Usage> {
        let mut file = match File::open(path) {
            Ok(file) => file,
            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
                *self = Self::default();
                return Ok(Usage::default());
            }
            Err(error) => return Err(error),
        };
        let length = file.metadata()?.len();
        let id = file_id(&file)?;
        if length < self.offset || self.file_id != Some(id) {
            self.offset = 0;
            self.days.clear();
            self.all = Totals::default();
        }
        self.file_id = Some(id);
        file.seek(SeekFrom::Start(self.offset))?;

        let week_start = today - chrono::Duration::days(6);
        self.days.retain(|day, _| *day >= week_start);
        let mut reader = BufReader::new(file);
        let mut line = Vec::new();
        loop {
            line.clear();
            let bytes = reader.read_until(b'\n', &mut line)?;
            // The offset stays before an unfinished line so the next refresh retries it.
            if line.last() != Some(&b'\n') {
                break;
            }
            let invalid = |message| {
                std::io::Error::new(
                    std::io::ErrorKind::InvalidData,
                    format!("invalid routing record at byte {}: {message}", self.offset),
                )
            };
            let Some(record) = parse_entry(&line).map_err(|error| invalid(error.to_string()))?
            else {
                self.offset += bytes as u64;
                continue;
            };
            if record.model.trim().is_empty() {
                return Err(invalid("model must not be empty".into()));
            }
            let ts = DateTime::parse_from_rfc3339(&record.ts)
                .map_err(|error| invalid(format!("invalid timestamp: {error}")))?;
            self.offset += bytes as u64;
            let day = ts.with_timezone(&Local).date_naive();
            self.all.add(&record);
            if day >= week_start {
                self.days.entry(day).or_default().add(&record);
            }
        }
        let mut usage = Usage {
            all: self.all.clone(),
            ..Usage::default()
        };
        for (day, totals) in &self.days {
            if *day == today {
                usage.today = totals.clone();
            }
            if *day <= today {
                usage.week.merge(totals);
                usage.days.insert(*day, totals.clone());
            }
        }
        Ok(usage)
    }
}

/// This function returns the device and inode numbers that identify an open file.
/// A replaced log has a new inode, so the reader starts its totals over.
#[cfg(unix)]
fn file_id(file: &File) -> std::io::Result<(u64, u64)> {
    use std::os::unix::fs::MetadataExt;
    let metadata = file.metadata()?;
    Ok((metadata.dev(), metadata.ino()))
}

/// This function returns the volume serial number and file index that identify an open file.
/// NTFS can give a recreated file the old creation time, so creation time cannot detect a
/// replaced log.
#[cfg(windows)]
fn file_id(file: &File) -> std::io::Result<(u64, u64)> {
    use std::os::windows::io::AsRawHandle;
    use windows_sys::Win32::Storage::FileSystem::{
        BY_HANDLE_FILE_INFORMATION, GetFileInformationByHandle,
    };
    let mut info = BY_HANDLE_FILE_INFORMATION::default();
    // The handle stays open for the call because `file` borrows it.
    if unsafe { GetFileInformationByHandle(file.as_raw_handle(), &mut info) } == 0 {
        return Err(std::io::Error::last_os_error());
    }
    let index = (u64::from(info.nFileIndexHigh) << 32) | u64::from(info.nFileIndexLow);
    Ok((u64::from(info.dwVolumeSerialNumber), index))
}

#[cfg(test)]
mod tests {
    use super::*;

    fn date(text: &str) -> NaiveDate {
        text.parse().expect("valid date")
    }

    /// This function builds a log line stamped at a local time, mirroring how the server
    /// writes timestamps for events that happened in the local day.
    fn line(local_ts: &str, model: &str, tier: &str, prompt: u64, completion: u64) -> String {
        let offset = Local::now().offset().to_string();
        format!(
            r#"{{"ts":"{local_ts}{offset}","model":"{model}","tier":"{tier}","prompt_tokens":{prompt},"cached_tokens":0,"completion_tokens":{completion},"reasoning_tokens":0}}"#
        )
    }

    fn log(contents: &str) -> (tempfile::TempDir, std::path::PathBuf) {
        let dir = tempfile::tempdir().expect("tempdir");
        let path = dir.path().join("routing.jsonl");
        std::fs::write(&path, contents).expect("write log");
        (dir, path)
    }

    #[test]
    fn reads_calls_among_outcomes_and_retries_an_unfinished_line() {
        use std::io::Write;

        let outcome = r#"{"session_id":"s","outcome":{"ts":"2026-09-28T10:00:00Z","route_id":"smart","judgments":[]}}"#;
        let first = line("2026-09-28T10:00:00.000", "luna", "", 100, 10);
        let second = line("2026-09-28T10:00:01.000", "terra", "classifier", 30, 1);
        let (_dir, path) = log(&format!("{outcome}\n{first}\n{outcome}\n"));
        let today = date("2026-09-28");
        let mut reader = Reader::default();
        assert_eq!(reader.read(&path, today).expect("read").today.tokens(), 110);
        assert_eq!(
            reader.read(&path, today).expect("reread").today.tokens(),
            110
        );

        let mut file = std::fs::OpenOptions::new()
            .append(true)
            .open(&path)
            .expect("open");
        let split = second.len() / 2;
        file.write_all(&second.as_bytes()[..split])
            .expect("partial append");
        assert_eq!(
            reader
                .read(&path, today)
                .expect("partial read")
                .today
                .tokens(),
            110
        );
        writeln!(file, "{}", &second[split..]).expect("finish append");
        writeln!(file, "{outcome}").expect("append outcome");
        let usage = reader.read(&path, today).expect("read append");
        assert_eq!(usage.today.tokens(), 141);
        assert_eq!(usage.today.requests(), 1);
        assert_eq!(usage.today.classifier["terra"].requests, 1);
        assert_eq!(reader.read(&path, today).expect("reread").all, usage.all);
    }

    #[test]
    fn rejects_invalid_calls_even_when_they_contain_an_outcome() {
        for contents in [
            "not json\n",
            "{\"outcome\":null}\n",
            "{\"outcome\":{},\"prompt_tokens\":100}\n",
            "{\"outcome\":{},\"cost\":{}}\n",
            "{\"outcome\":{},\"ts\":\"2026-09-28T10:00:00Z\",\"model\":42}\n",
        ] {
            let (_dir, path) = log(contents);
            let error = Reader::default()
                .read(&path, date("2026-09-28"))
                .expect_err("invalid call");
            assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
            assert!(
                error
                    .to_string()
                    .contains("invalid routing record at byte 0")
            );
        }
    }

    #[test]
    fn resets_totals_when_the_log_is_replaced_or_truncated() {
        let (_dir, path) = log(&format!(
            "{}\n{}\n",
            line("2026-09-28T10:00:00.000", "luna", "", 100, 10),
            line("2026-09-28T10:00:01.000", "luna", "", 100, 10),
        ));
        let mut reader = Reader::default();
        let today = date("2026-09-28");
        assert_eq!(reader.read(&path, today).expect("read").today.requests(), 2);
        let replacement = format!("{}\n", line("2026-09-28T11:00:00.000", "sol", "", 200, 20));
        std::fs::write(&path, &replacement).expect("truncate");
        let usage = reader.read(&path, today).expect("read truncated");
        assert_eq!(usage.today.requests(), 1);
        assert_eq!(usage.today.tokens(), 220);
        assert_eq!(usage.all.tokens(), 220);

        // The new first record has the same length as the old log but different counts,
        // so a reader that kept its offset would skip it and keep the old 220 tokens.
        std::fs::rename(&path, path.with_extension("old")).expect("rotate");
        let first = format!("{}\n", line("2026-09-28T11:00:00.000", "sol", "", 400, 40));
        assert_eq!(first.len(), replacement.len());
        std::fs::write(&path, first + &replacement.repeat(2)).expect("replace");
        let usage = reader.read(&path, today).expect("read replacement");
        assert_eq!(usage.today.requests(), 3);
        assert_eq!(usage.today.tokens(), 880);
        assert_eq!(usage.all.tokens(), 880);
        std::fs::remove_file(&path).expect("remove log");
        let absent = reader.read(&path, today).expect("missing log");
        assert!(absent.today.is_empty() && absent.week.is_empty() && absent.all.is_empty());
    }
}