Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
//! This module lists the models an LLM client offers from its `GET /models` endpoint,
//! and keeps each fetched list in a cache file.
//!
//! The request goes through the system `curl`, the same way the app runs
//! `launchctl`, so the app needs no HTTP or TLS stack of its own. The key is
//! written to curl's stdin, never to its command line, where other processes
//! could read it.
//!
//! The cache file holds model IDs and fetch times, never a key. The app uses
//! a list from the cache file until the user refreshes it, so it fetches a
//! list on its own only when the file does not have it.
use std::collections::BTreeMap;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::sync::{Mutex, PoisonError};
use std::time::{SystemTime, UNIX_EPOCH};
use serde::{Deserialize, Serialize};
use crate::config::expand_home;
use crate::server_config::Client;
/// CACHE_FILE names the cache file. It sits next to the settings file that the
/// app was started with.
pub const CACHE_FILE: &str = "model-lists.json";
/// KEYCHAIN_SERVICE identifies keys saved from Routes. Each
/// item's account is the client's `base_url`, so clients that share a
/// gateway share its key.
#[cfg(target_os = "macos")]
const KEYCHAIN_SERVICE: &str = "Switchyard model list";
/// TIMEOUT_SECONDS limits how long curl may spend listing models.
const TIMEOUT_SECONDS: &str = "20";
/// CACHE_WRITE stays locked while a thread rewrites the cache file, so threads that
/// finish at the same time do not drop each other's lists.
static CACHE_WRITE: Mutex<()> = Mutex::new(());
/// ModelList stores one fetched model list in the cache file.
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct ModelList {
/// This field stores sorted model IDs.
pub models: Vec<String>,
/// This field records the fetch time in seconds since 1970-01-01 UTC.
pub fetched_at: u64,
}
/// Loaded stores the model list and fetch result for one endpoint.
#[derive(Clone, Debug, PartialEq)]
pub struct Loaded {
/// This field identifies an endpoint list as `endpoint` or a Switchyard cache list as `cache`.
/// A Codex cache list has the source `local`; `error` separately reports fetch or cache-write failures.
pub source: &'static str,
/// This field stores the newest list fetched or read from the cache file.
pub list: Option<ModelList>,
/// This field records a fetch or cache-write error, if either occurred.
pub error: Option<ListError>,
}
/// ListError describes why a client's models could not be listed.
#[derive(Clone, Debug, PartialEq)]
pub enum ListError {
/// The client needs a key, and none is set or saved.
NoKey,
/// The client reads its key from this `api_key_env` variable, which this
/// process does not have, and no key is saved.
NoEnv(String),
/// The platform's credential store did not return the saved key.
Keychain(String),
/// The models endpoint answered 401 or 403.
Rejected(u16),
/// The request failed, or the answer was not a model list.
Failed(String),
/// The list was fetched, but the cache file could not be written.
NotCached(String),
}
impl ListError {
/// This function says what went wrong, in one sentence.
pub fn reason(&self) -> String {
match self {
Self::NoKey => {
"This endpoint needs an API key to list models, and the app does not have one."
.to_string()
}
Self::NoEnv(variable) => missing_env_note(variable),
Self::Keychain(error) => {
format!("Could not read the saved key from secure credential storage: {error}.")
}
Self::Rejected(status) => {
format!("The models endpoint rejected the key (HTTP {status}).")
}
Self::Failed(error) => format!("Could not list models: {error}."),
Self::NotCached(error) => format!("Could not save the list: {error}."),
}
}
/// This function says what the user can do next on the Routes page, where `m` loads
/// models again and `k` opens the form that saves a model-list key. Only macOS and
/// Windows can save a key, so elsewhere a key error gets no advice.
/// [`missing_env_note`] already explains a missing variable, so that error gets no advice.
pub fn advice(&self) -> &'static str {
let saves_keys = cfg!(any(target_os = "macos", windows));
match self {
Self::NoEnv(_) => "",
_ if self.needs_key() && saves_keys => "Press k on Routes to save a valid key.",
_ if self.needs_key() => "",
_ => "Press m on Routes to try again.",
}
}
/// This function returns whether saving a key could fix this error.
pub fn needs_key(&self) -> bool {
matches!(
self,
Self::NoKey | Self::NoEnv(_) | Self::Keychain(_) | Self::Rejected(_)
)
}
}
/// This function explains why model suggestions lack a key. Model-list keys stay separate from
/// the server keys that the installed helper keeps.
pub fn missing_env_note(variable: &str) -> String {
if cfg!(any(target_os = "macos", windows)) {
format!(
"Switchyard does not read {variable} from your shell. \
To load this endpoint's models, press k on Routes to save a model-list key. \
The server's own key is under Settings → Server keys."
)
} else {
format!(
"{variable} is not set in this terminal, so this endpoint cannot list models. \
Set {variable} and start the app again."
)
}
}
/// This function returns the client's `api_key_env` variable when this process does not
/// have it set.
pub fn missing_env(client: &Client) -> Option<&str> {
client
.api_key_env
.as_deref()
.filter(|_| env_key(client).is_none())
}
/// This function loads one endpoint's models and retains the cached list when refresh fails.
/// The selected endpoint alone receives the entered key.
pub fn load(cache: &Path, client: &Client, refresh: bool, typed_key: Option<&str>) -> Loaded {
if unlisted(client) {
// ChatGPT has no list to fetch, so nothing goes in the cache file.
return Loaded {
source: "local",
list: codex_models(),
error: None,
};
}
let url = match models_url(&client.format, &client.base_url) {
Ok(url) => url,
Err(error) => {
return Loaded {
source: "cache",
list: None,
error: Some(ListError::Failed(error)),
};
}
};
let old = read_cache(cache).remove(&url);
if old.is_some() && !refresh {
return Loaded {
source: "cache",
list: old,
error: None,
};
}
match key(client, typed_key).and_then(|key| list(client, &url, key.as_deref())) {
Ok(models) => {
let new = ModelList {
models,
fetched_at: now(),
};
let error = write_cache(cache, &url, &new)
.err()
.map(ListError::NotCached);
Loaded {
source: "endpoint",
list: Some(new),
error,
}
}
Err(error) => Loaded {
source: "cache",
list: old,
error: Some(error),
},
}
}
/// This function returns whether the client's endpoint has no model list that this app can
/// ask for. The ChatGPT Codex endpoint answers only a ChatGPT login, never an
/// API key, and the app does not hold the login.
pub fn unlisted(client: &Client) -> bool {
client.host() == "chatgpt.com"
}
/// This function reads the model list that Codex saved in `$CODEX_HOME/models_cache.json`,
/// or in `~/.codex/models_cache.json` when `CODEX_HOME` is unset. Codex
/// fetched that list with the user's ChatGPT login. For a ChatGPT client this
/// file is the only list of its models that the app can read.
fn codex_models() -> Option<ModelList> {
let home = std::env::var_os("CODEX_HOME")
.map(PathBuf::from)
.or_else(|| expand_home(Path::new("~/.codex")).ok())?;
read_codex_models(&home.join("models_cache.json"))
}
/// This function parses a Codex model list file. Returns every model's `slug`, sorted and
/// without duplicates, and the time Codex fetched the file.
fn read_codex_models(path: &Path) -> Option<ModelList> {
#[derive(Deserialize)]
struct Saved {
fetched_at: String,
models: Vec<Entry>,
}
#[derive(Deserialize)]
struct Entry {
slug: String,
}
let saved: Saved = serde_json::from_str(&std::fs::read_to_string(path).ok()?).ok()?;
let fetched_at = chrono::DateTime::parse_from_rfc3339(&saved.fetched_at).ok()?;
let mut models: Vec<String> = saved.models.into_iter().map(|entry| entry.slug).collect();
models.sort();
models.dedup();
Some(ModelList {
models,
fetched_at: u64::try_from(fetched_at.timestamp()).ok()?,
})
}
/// This function lists the model IDs at `url`, sorted. When the client needs a key and
/// `key` is `None`, returns [`ListError::NoEnv`] or [`ListError::NoKey`]
/// without sending a request.
fn list(client: &Client, url: &str, key: Option<&str>) -> Result<Vec<String>, ListError> {
match (key, &client.api_key_env) {
(None, Some(variable)) => Err(ListError::NoEnv(variable.clone())),
(None, None) if client.forward_auth => Err(ListError::NoKey),
_ => fetch(url, &client.format, key),
}
}
/// This function returns the models endpoint for a client. It builds the URL the way the
/// server builds its request URLs: it parses `base_url` and changes only the path.
/// OpenAI clients drop a trailing `/chat/completions` or `/responses`. Anthropic clients
/// drop a trailing `/v1/messages` to `/v1`, and add `/v1` to any other path.
/// Anthropic returns 20 models unless the URL sets `limit`, so the URL adds
/// `limit=1000`, the most that Anthropic allows. The URL is also the list's key in
/// the cache file. curl never sends a fragment, so the URL drops it.
fn models_url(format: &str, base_url: &str) -> Result<String, String> {
let mut url = url::Url::parse(base_url)
.map_err(|error| format!("the base URL {base_url} is not valid: {error}"))?;
let base = url.path().trim_end_matches('/');
let path = if format == "anthropic_messages" {
let root = base
.strip_suffix("/messages")
.filter(|root| root.ends_with("/v1"))
.unwrap_or(base);
if root.ends_with("/v1") {
format!("{root}/models")
} else {
format!("{root}/v1/models")
}
} else {
let root = base
.strip_suffix("/chat/completions")
.or_else(|| base.strip_suffix("/responses"))
.unwrap_or(base);
format!("{root}/models")
};
url.set_path(&path);
url.set_fragment(None);
if format == "anthropic_messages" {
url.query_pairs_mut().append_pair("limit", "1000");
}
Ok(url.into())
}
/// This function returns whether the server sends this client a key. If it does, listing
/// the client's models needs a key too.
fn sends_key(client: &Client) -> bool {
client.forward_auth || client.api_key_env.is_some()
}
/// This function returns the key to list the client's models with, or `None` when the
/// client needs no key or none is available.
///
/// The entered key comes first, then the key saved from Routes, then the client's
/// `api_key_env` variable. The saved key wins over the variable because the app checked
/// it against the endpoint when the user saved it. Otherwise an expired key in the
/// variable would hide every replacement that the user saves.
fn key(client: &Client, typed_key: Option<&str>) -> Result<Option<String>, ListError> {
if !sends_key(client) {
return Ok(None);
}
if let Some(key) = typed_key {
return Ok(Some(key.to_string()));
}
let saved = saved_key(&client.base_url).map_err(ListError::Keychain)?;
Ok(saved.or_else(|| env_key(client)))
}
fn env_key(client: &Client) -> Option<String> {
client
.api_key_env
.as_deref()
.and_then(|variable| std::env::var(variable).ok())
.filter(|key| !key.trim().is_empty())
}
/// This function returns the key saved in the Keychain for `base_url`, or `None` when no
/// key is saved.
#[cfg(target_os = "macos")]
fn saved_key(base_url: &str) -> Result<Option<String>, String> {
/// NOT_FOUND matches the Keychain's `errSecItemNotFound` status.
const NOT_FOUND: i32 = -25300;
match security_framework::passwords::get_generic_password(KEYCHAIN_SERVICE, base_url) {
Ok(bytes) => String::from_utf8(bytes)
.map(|key| Some(key).filter(|key| !key.trim().is_empty()))
.map_err(|_| "the saved key is not text".to_string()),
Err(error) if error.code() == NOT_FOUND => Ok(None),
Err(error) => Err(keychain_error(&error)),
}
}
#[cfg(not(any(target_os = "macos", windows)))]
fn saved_key(_base_url: &str) -> Result<Option<String>, String> {
Ok(None)
}
/// This function saves a key in the login Keychain for every client with this `base_url`.
#[cfg(target_os = "macos")]
pub fn save_key(base_url: &str, key: &str) -> Result<(), String> {
security_framework::passwords::set_generic_password(
KEYCHAIN_SERVICE,
base_url,
key.trim().as_bytes(),
)
.map_err(|error| keychain_error(&error))
}
#[cfg(not(any(target_os = "macos", windows)))]
/// This function rejects saved keys on platforms without Keychain or Credential Manager support.
pub fn save_key(_base_url: &str, _key: &str) -> Result<(), String> {
Err(
"Saving model-list keys requires macOS Keychain or Windows Credential Manager. \
On this platform, set the environment variable named by api_key_env \
and make it available to the app."
.into(),
)
}
#[cfg(windows)]
/// This function reads a saved key for this exact base_url from Windows Credential Manager.
fn saved_key(base_url: &str) -> Result<Option<String>, String> {
switchyard_desktop::install::windows::saved_key(base_url)
}
#[cfg(windows)]
/// This function saves the key for this exact base_url in Windows Credential Manager.
pub fn save_key(base_url: &str, key: &str) -> Result<(), String> {
switchyard_desktop::install::windows::save_key(base_url, key)
}
/// This function returns the Keychain's message without its final period, so the window
/// can end its own sentence after it.
#[cfg(target_os = "macos")]
fn keychain_error(error: &security_framework::base::Error) -> String {
error.to_string().trim_end_matches('.').to_string()
}
/// This function returns whether a key has a line break, which would end the header that
/// carries the key and start another one.
pub fn has_line_break(key: &str) -> bool {
key.contains(['\r', '\n'])
}
/// This function reads the cache file. A missing or unreadable file reads as empty, so the
/// app fetches its lists again and writes a new file.
fn read_cache(path: &Path) -> BTreeMap<String, ModelList> {
std::fs::read_to_string(path)
.ok()
.and_then(|text| serde_json::from_str(&text).ok())
.unwrap_or_default()
}
/// This function adds the list at `url` to the cache file. The list replaces the one at
/// the same URL unless that one is newer: two loads can overlap, and the
/// load that fetched a list first can write it last. The text goes to a
/// temporary file that is renamed over the cache file, so a reader never
/// sees half a file.
fn write_cache(path: &Path, url: &str, list: &ModelList) -> Result<(), String> {
let _writing = CACHE_WRITE.lock().unwrap_or_else(PoisonError::into_inner);
let mut all = read_cache(path);
if all
.get(url)
.is_some_and(|cached| cached.fetched_at > list.fetched_at)
{
return Ok(());
}
all.insert(url.to_string(), list.clone());
let text = serde_json::to_string_pretty(&all)
.map_err(|error| format!("write {}: {error}", path.display()))?;
let dir = path
.parent()
.filter(|dir| !dir.as_os_str().is_empty())
.unwrap_or(Path::new("."));
let mut file = tempfile::NamedTempFile::new_in(dir)
.map_err(|error| format!("create a file in {}: {error}", dir.display()))?;
file.write_all(text.as_bytes())
.and_then(|()| file.as_file().sync_all())
.map_err(|error| format!("write {}: {error}", path.display()))?;
file.persist(path)
.map_err(|error| format!("replace {}: {}", path.display(), error.error))?;
Ok(())
}
/// This function returns the current time in seconds since 1970-01-01 UTC.
pub fn now() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_or(0, |since| since.as_secs())
}
fn fetch(url: &str, format: &str, key: Option<&str>) -> Result<Vec<String>, ListError> {
let failed = |error: String| ListError::Failed(error);
if key.is_some_and(has_line_break) {
return Err(failed(
"the key has a line break, so the app did not send it".to_string(),
));
}
let mut command = Command::new("curl");
#[cfg(windows)]
switchyard_desktop::install::windows::hide_console(&mut command);
let mut child = command
// curl honors `-q` only as its first argument. `-q` makes curl ignore
// ~/.curlrc, where a `verbose` line would print the key into the
// error text that the window shows.
.args([
"-q",
"--silent",
"--show-error",
"--max-time",
TIMEOUT_SECONDS,
])
// curl reads request headers from stdin, one per line.
.args(["--header", "@-"])
// `--url` keeps a URL that starts with `-` from being read as an
// option.
.args(["--write-out", "\n%{http_code}", "--url", url])
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.map_err(|error| failed(format!("run curl: {error}")))?;
if let Some(mut stdin) = child.stdin.take() {
let headers = match (key, format) {
(Some(key), "anthropic_messages") => {
format!("x-api-key: {key}\nanthropic-version: 2023-06-01\n")
}
(Some(key), _) => format!("Authorization: Bearer {key}\n"),
(None, _) => String::new(),
};
let sent = stdin.write_all(headers.as_bytes());
drop(stdin);
if let Err(error) = sent {
let _ = child.kill();
let _ = child.wait();
return Err(failed(format!("send headers to curl: {error}")));
}
}
let output = child
.wait_with_output()
.map_err(|error| failed(format!("run curl: {error}")))?;
if !output.status.success() {
return Err(failed(
String::from_utf8_lossy(&output.stderr).trim().to_string(),
));
}
let stdout = String::from_utf8_lossy(&output.stdout);
let (body, status) = stdout.rsplit_once('\n').unwrap_or(("", &stdout));
match status.trim().parse::<u16>() {
Ok(200) => parse_ids(body).map_err(failed),
Ok(status @ (401 | 403)) => Err(ListError::Rejected(status)),
Ok(status) => Err(failed(format!("{url} answered HTTP {status}"))),
Err(_) => Err(failed(format!("{url} sent no HTTP status"))),
}
}
/// This function parses the model IDs. OpenAI and Anthropic both answer with
/// `{"data": [{"id": ...}, ...]}`.
fn parse_ids(body: &str) -> Result<Vec<String>, String> {
#[derive(Deserialize)]
struct Listing {
data: Vec<Entry>,
}
#[derive(Deserialize)]
struct Entry {
id: String,
}
let listing: Listing = serde_json::from_str(body)
.map_err(|error| format!("the model list is not the expected JSON: {error}"))?;
let mut ids: Vec<String> = listing.data.into_iter().map(|entry| entry.id).collect();
ids.sort();
ids.dedup();
Ok(ids)
}
#[cfg(test)]
mod tests {
use super::models_url;
// A fragment or query in the base URL must not swallow the `/models` path.
#[test]
fn models_url_changes_only_the_path() {
for (format, base_url, expected) in [
(
"openai_responses",
"https://example.test/v1/responses#production",
"https://example.test/v1/models",
),
(
"openai_chat",
"https://example.test/v1/chat/completions?api-version=1",
"https://example.test/v1/models?api-version=1",
),
(
"anthropic_messages",
"https://example.test/v1/messages",
"https://example.test/v1/models?limit=1000",
),
(
"anthropic_messages",
"https://example.test?beta=1",
"https://example.test/v1/models?beta=1&limit=1000",
),
] {
assert_eq!(models_url(format, base_url).as_deref(), Ok(expected));
}
assert!(models_url("openai_chat", "example.test/v1").is_err());
}
}