Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(vgr): discover local backend context capacity (#992)

GDRSbca35291a698
diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs
index 2db3950bf..4c3d82a82 100644
--- a/crates/libsy-llm-client/src/client.rs
+++ b/crates/libsy-llm-client/src/client.rs
@@ -200,6 +200,56 @@ impl TranslatingLlmClient {
self.backend_for(model, operation.wire_format()).is_some()
}
+ /// Reads llama.cpp model properties through the backend's configured auth and headers.
+ ///
+ /// Redirects are disabled so credentials cannot move to another origin.
+ pub async fn get_model_properties(
+ &self,
+ model: &ModelId,
+ format: WireFormat,
+ timeout: Duration,
+ ) -> Result<Value> {
+ let backend =
+ self.backend_for(model, format)
+ .ok_or_else(|| LlmClientError::Configuration {
+ message: format!("model {model} has no backend for {format:?}"),
+ })?;
+ let mut url = reqwest::Url::parse(backend.base_url()).map_err(|error| {
+ LlmClientError::Configuration {
+ message: format!("model {model} has an invalid backend URL: {error}"),
+ }
+ })?;
+ let root = url.path().trim_end_matches('/');
+ let root = root
+ .strip_suffix("/chat/completions")
+ .or_else(|| root.strip_suffix("/responses"))
+ .unwrap_or(root);
+ let path = format!(
+ "{}/props",
+ root.strip_suffix("/v1")
+ .unwrap_or(root)
+ .trim_end_matches('/')
+ );
+ url.set_path(&path);
+ url.set_query(None);
+
+ let builder = self.forward_auth_client.get(url).timeout(timeout);
+ let builder = apply_extra_headers(builder, backend);
+ let response = backend
+ .apply_auth(builder)
+ .send()
+ .await
+ .map_err(convert_reqwest_error)?
+ .error_for_status()
+ .map_err(convert_reqwest_error)?;
+ response
+ .json()
+ .await
+ .map_err(|source| LlmClientError::InvalidResponse {
+ source: Box::new(source),
+ })
+ }
+
/// Calls a model-bearing auxiliary provider operation.
///
/// Returns an error when the model has no compatible backend or the upstream
@@ -1519,6 +1569,43 @@ mod tests {
)]
}
+ #[tokio::test]
+ async fn model_properties_uses_configured_auth_and_backend_path()
+ -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
+ for (base_path, props_path) in [
+ ("/v1", "/props"),
+ ("/v1/", "/props"),
+ ("/v1/chat/completions", "/props"),
+ ("/v1/responses/", "/props"),
+ ("/prefix/v1", "/prefix/props"),
+ ("/prefix/v1/chat/completions", "/prefix/props"),
+ ("/prefix/v1/responses", "/prefix/props"),
+ ("/prefix/", "/prefix/props"),
+ ] {
+ let server = MockServer::start().await;
+ Mock::given(method("GET"))
+ .and(path(props_path))
+ .and(wiremock::matchers::header("authorization", "Bearer secret"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({
+ "default_generation_settings": {"n_ctx": 65_536}
+ })))
+ .expect(1)
+ .mount(&server)
+ .await;
+ let client =
+ TranslatingLlmClient::new(&chat_map(&format!("{}{base_path}", server.uri())))?;
+ let properties = client
+ .get_model_properties(
+ &ModelId::from("gpt"),
+ WireFormat::OpenAiChat,
+ Duration::from_secs(1),
+ )
+ .await?;
+ assert_eq!(properties["default_generation_settings"]["n_ctx"], 65_536);
+ }
+ Ok(())
+ }
+
fn chat_map_with_retries(base_url: &str, max_retries: u32) -> Vec<ModelConfig> {
vec![ModelConfig::new(
"gpt",
diff --git a/crates/libsy/src/algorithms/vgr/readout.rs b/crates/libsy/src/algorithms/vgr/readout.rs
index d9f4491b9..62533af93 100644
--- a/crates/libsy/src/algorithms/vgr/readout.rs
+++ b/crates/libsy/src/algorithms/vgr/readout.rs
@@ -47,8 +47,6 @@ pub(super) fn p_yes(response: &AggLlmResponse) -> Option<f64> {
}
match (yes, no) {
(Some(yes), Some(no)) if yes + no > 0.0 => Some(yes / (yes + no)),
- (Some(_), None) => Some(1.0),
- (None, Some(_)) => Some(0.0),
_ => None,
}
}
@@ -75,10 +73,8 @@ mod tests {
{"token": ".", "logprob": -9.0}
]));
assert!(score.is_some_and(|score| (score - 0.5).abs() < 1e-9));
- assert_eq!(
- scored(json!([{"token": "yes", "logprob": -3.0}])),
- Some(1.0)
- );
+ assert_eq!(scored(json!([{"token": "yes", "logprob": -3.0}])), None);
+ assert_eq!(scored(json!([{"token": "no", "logprob": -3.0}])), None);
assert_eq!(scored(json!([{"token": "maybe", "logprob": 0.0}])), None);
assert_eq!(p_yes(&AggLlmResponse::default()), None);
}
diff --git a/crates/libsy/src/algorithms/vgr/runtime/tests.rs b/crates/libsy/src/algorithms/vgr/runtime/tests.rs
index ea2f6fdb6..8bdd7f4bd 100644
--- a/crates/libsy/src/algorithms/vgr/runtime/tests.rs
+++ b/crates/libsy/src/algorithms/vgr/runtime/tests.rs
@@ -203,13 +203,13 @@ async fn final_answers_commit_only_on_a_verified_clean_record() -> Result<()> {
long.extend([call(&id), result(&id, "ok")]);
}
assert_eq!(
- drive(&clean, session(long.clone()), calls.clone(), answer, 1.0).await?,
+ drive(&clean, session(long.clone()), calls.clone(), answer, 0.99).await?,
"cloud"
);
assert_eq!(calls.1.load(Ordering::Relaxed), 0);
let confirmed = route(|config| config.confirmed_recovery_min_clean_tail = Some(1))?;
assert_eq!(
- drive(&confirmed, session(long), calls.clone(), answer, 1.0).await?,
+ drive(&confirmed, session(long), calls.clone(), answer, 0.99).await?,
"local"
);
assert_eq!(calls.1.load(Ordering::Relaxed), 1);
diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs
index 82010f6c3..1b4b2fc8a 100644
--- a/crates/switchyard-runner/src/config.rs
+++ b/crates/switchyard-runner/src/config.rs
@@ -18,6 +18,7 @@ use switchyard_llm_client::{
};
use switchyard_protocol::{ModelId, RoutedDecisionClient, RoutedLlmClient, WireFormat};
+use crate::runner::ModelPropertiesProbe;
use crate::{
AlgorithmSpec, AuxiliaryTarget, CallerAuthKind, DecisionTarget, ModelCapabilities, Route,
Runner, RunnerError,
@@ -317,6 +318,7 @@ impl DeploymentConfig {
let targets = self.build_targets();
let fallback_base_url = self.fallback_base_url()?;
let mut routes = Vec::with_capacity(self.routes.len());
+ let mut model_properties_probes = BTreeMap::new();
for (route_name, config) in &self.routes {
let decision_judge = config.algorithm.decision_judge();
for name in config.callable_target_names() {
@@ -352,6 +354,11 @@ impl DeploymentConfig {
.algorithm
.build_with_runtime_models(route_name, &targets)
.map_err(|error| RunnerError::configuration_source(error.to_string(), error))?;
+ if let AlgorithmSpec::Vgr { config: vgr } = &config.algorithm
+ && let Some(probe) = self.build_model_properties_probe(&vgr.local_target, &clients)
+ {
+ model_properties_probes.insert(config.id.clone(), probe);
+ }
let (route_clients, caller_auth) =
self.build_route_clients(route_name, config, &clients, &decision_clients)?;
let anthropic_auxiliary_target =
@@ -377,7 +384,8 @@ impl DeploymentConfig {
}
let runner = Runner::new(routes)
.with_fallback_url(fallback_base_url)
- .with_provider_api_keys(provider_api_keys);
+ .with_provider_api_keys(provider_api_keys)
+ .with_model_properties_probes(model_properties_probes);
Ok(runner)
}
@@ -670,6 +678,27 @@ impl DeploymentConfig {
Ok(Some(config.base_url.as_str().to_string()))
}
+ fn build_model_properties_probe(
+ &self,
+ local_target_name: &str,
+ clients: &BTreeMap<String, Arc<TranslatingLlmClient>>,
+ ) -> Option<ModelPropertiesProbe> {
+ let local_target = self.targets.get(local_target_name)?;
+ let client_config = self.llm_clients.get(&local_target.llm_client)?;
+ if !matches!(
+ client_config.context_discovery,
+ Some(ContextDiscovery::LlamaCpp)
+ ) {
+ return None;
+ }
+ let client = clients.get(&local_target.llm_client)?;
+ Some(ModelPropertiesProbe {
+ model: local_target.id.clone(),
+ format: client_config.format.wire_format(),
+ client: Arc::clone(client),
+ })
+ }
+
fn build_anthropic_auxiliary_target(
&self,
route: &RouteConfig,
@@ -763,6 +792,8 @@ impl<'de> Deserialize<'de> for HttpBaseUrl {
struct LlmClientConfig {
format: ClientFormat,
base_url: HttpBaseUrl,
+ /// Opt-in context discovery for VGR's local target. Unset disables discovery.
+ context_discovery: Option<ContextDiscovery>,
api_key_env: Option<String>,
#[serde(default)]
forward_auth: bool,
@@ -779,6 +810,12 @@ struct LlmClientConfig {
timeout_ms: Option<u64>,
}
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "snake_case")]
+enum ContextDiscovery {
+ LlamaCpp,
+}
+
#[derive(Debug, Deserialize)]
#[serde(tag = "format", rename_all = "snake_case", deny_unknown_fields)]
enum DecisionClientConfig {
@@ -1192,6 +1229,19 @@ task_typing = false
Ok(())
}
+ #[test]
+ fn vgr_local_server_example_builds_and_rejects_unknown_discovery() -> RunnerResult<()> {
+ let example = include_str!("../../../examples/vgr-local-servers.toml");
+ let runner = runner_from_toml(example)?;
+ assert!(runner.route("switchyard/vgr").is_some());
+ let invalid = example.replace(
+ "context_discovery = \"llama_cpp\"",
+ "context_discovery = \"guess\"",
+ );
+ assert!(error_message(&invalid).contains("unknown variant `guess`"));
+ Ok(())
+ }
+
fn with_subagent_llm_classifier(config: &str, route: &str, extra: &str) -> String {
let mut configured = config.to_string();
configured.push_str(&format!("\n[routes.{route}.subagents]\n"));
diff --git a/crates/switchyard-runner/src/runner.rs b/crates/switchyard-runner/src/runner.rs
index ffba5120f..e71fe204c 100644
--- a/crates/switchyard-runner/src/runner.rs
+++ b/crates/switchyard-runner/src/runner.rs
@@ -5,9 +5,12 @@
use std::collections::BTreeMap;
use std::path::Path;
+use std::sync::Arc;
+use std::time::Duration;
use libsy::RoutingOutcome;
use serde_json::Value;
+use switchyard_llm_client::TranslatingLlmClient;
use switchyard_protocol::{ModelId, WireFormat};
use crate::config;
@@ -18,6 +21,7 @@ pub struct Runner {
routes: Vec<(ModelId, Route)>,
fallback_base_url: Option<String>,
provider_api_keys: Vec<String>,
+ model_properties_probes: BTreeMap<ModelId, ModelPropertiesProbe>,
}
/// Borrowed model metadata returned while listing routes.
@@ -43,6 +47,12 @@ pub struct DecisionTarget {
pub extra_body: BTreeMap<String, Value>,
}
+pub(crate) struct ModelPropertiesProbe {
+ pub(crate) model: ModelId,
+ pub(crate) format: WireFormat,
+ pub(crate) client: Arc<TranslatingLlmClient>,
+}
+
impl Runner {
/// Loads and validates a version-1 deployment TOML file.
pub fn load(path: impl AsRef<Path>) -> Result<Self, RunnerError> {
@@ -72,6 +82,7 @@ impl Runner {
routes,
fallback_base_url: None,
provider_api_keys: Vec::new(),
+ model_properties_probes: BTreeMap::new(),
}
}
@@ -92,6 +103,14 @@ impl Runner {
self
}
+ pub(crate) fn with_model_properties_probes(
+ mut self,
+ probes: BTreeMap<ModelId, ModelPropertiesProbe>,
+ ) -> Self {
+ self.model_properties_probes = probes;
+ self
+ }
+
/// Returns the route registered for a model.
pub fn route(&self, model: &str) -> Option<&Route> {
self.routes
@@ -109,6 +128,16 @@ impl Runner {
})
}
+ /// Reads one route's configured local backend properties without exposing its credentials.
+ pub async fn model_properties(&self, model: &str, timeout: Duration) -> Option<Value> {
+ let probe = self.model_properties_probes.get(model)?;
+ probe
+ .client
+ .get_model_properties(&probe.model, probe.format, timeout)
+ .await
+ .ok()
+ }
+
/// Returns the validated API root used for unmatched HTTP requests.
pub fn fallback_base_url(&self) -> Option<&str> {
self.fallback_base_url.as_deref()
diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs
index 22b401788..9e3ac630a 100644
--- a/crates/switchyard-server/src/lib.rs
+++ b/crates/switchyard-server/src/lib.rs
@@ -169,6 +169,7 @@ pub struct ServerState {
stats: StatsAccumulator,
routing_log: Option<SharedRoutingLog>,
track_cache_eligibility: bool,
+ model_catalog: Arc<tokio::sync::Mutex<Option<(Instant, Value)>>>,
}
#[derive(Clone)]
@@ -226,6 +227,7 @@ impl ServerState {
stats,
routing_log: None,
track_cache_eligibility: tracking_enabled_from_env(),
+ model_catalog: Arc::default(),
})
}
@@ -1471,12 +1473,43 @@ fn error_response(
}
async fn models(State(state): State<ServerState>) -> Json<Value> {
- Json(model_list_payload(
- state
- .runner
- .models()
+ let mut cache = state.model_catalog.lock().await;
+ if let Some((updated, catalog)) = cache.as_ref()
+ && updated.elapsed() < Duration::from_secs(60)
+ {
+ return Json(catalog.clone());
+ }
+ let mut models: Vec<_> = state.runner.models().collect();
+ let probes = models
+ .iter_mut()
+ .filter(|model| model.capabilities.context_window.is_none())
+ .map(|model| async {
+ model.capabilities.context_window =
+ discover_context_window(&state.runner, model.id).await;
+ });
+ // Keep completed discoveries even when another backend exhausts the shared budget.
+ let _ = tokio::time::timeout(
+ Duration::from_secs(2),
+ futures_util::future::join_all(probes),
+ )
+ .await;
+ let catalog = model_list_payload(
+ models
+ .into_iter()
.map(|model| (model.id.as_str(), model.capabilities)),
- ))
+ );
+ *cache = Some((Instant::now(), catalog.clone()));
+ Json(catalog)
+}
+
+async fn discover_context_window(runner: &Runner, model: &ModelId) -> Option<u32> {
+ let body = runner
+ .model_properties(model.as_str(), Duration::from_secs(2))
+ .await?;
+ let context = body
+ .pointer("/default_generation_settings/n_ctx")?
+ .as_u64()?;
+ u32::try_from(context).ok().filter(|context| *context > 0)
}
async fn get_stats(State(state): State<ServerState>) -> Json<StatsSnapshot> {
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index 6e6ad34b3..315d2470c 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -1683,20 +1683,117 @@ impl Response {
}
}
+#[tokio::test]
+async fn models_probes_routes_concurrently_and_caches_failed_discovery() -> TestResult {
+ for stalled in [false, true] {
+ let calls = Arc::new(Mutex::new(Vec::new()));
+ let barrier = Arc::new(tokio::sync::Barrier::new(3));
+ let captured = Arc::clone(&calls);
+ let upstream_app = Router::new().route(
+ "/props",
+ get(move || {
+ let barrier = Arc::clone(&barrier);
+ let calls = Arc::clone(&captured);
+ async move {
+ calls.lock().await.push(json!({"path": "/props"}));
+ barrier.wait().await;
+ if stalled {
+ std::future::pending::<()>().await;
+ }
+ StatusCode::SERVICE_UNAVAILABLE
+ }
+ }),
+ );
+ let listener = TcpListener::bind("127.0.0.1:0").await?;
+ let addr = listener.local_addr()?;
+ let upstream = MockUpstream {
+ base_url: format!("http://{addr}/v1"),
+ calls,
+ task: tokio::spawn(async move {
+ axum::serve(listener, upstream_app).await.unwrap();
+ }),
+ };
+ let routes: String = (0..3)
+ .map(|index| {
+ format!(
+ r#"
+[routes.vgr{index}]
+id = "switchyard/vgr{index}"
+type = "vgr"
+local_target = "local"
+cloud_target = "cloud"
+"#
+ )
+ })
+ .collect();
+ let app = build_switchyard_router(load_test_config(&format!(
+ r#"
+schema_version = 1
+[llm_clients.local]
+format = "openai_chat"
+base_url = "{}"
+context_discovery = "llama_cpp"
+[targets.local]
+id = "local"
+llm_client = "local"
+[targets.cloud]
+id = "cloud"
+llm_client = "local"
+{routes}
+"#,
+ upstream.base_url
+ ))?);
+
+ let (first, concurrent) = tokio::time::timeout(std::time::Duration::from_secs(3), async {
+ tokio::join!(
+ send(&app, "GET", "/v1/models", None),
+ send(&app, "GET", "/v1/models", None)
+ )
+ })
+ .await?;
+ let models = first?.json()?;
+ assert_eq!(concurrent?.json()?, models);
+ assert_eq!(models["data"].as_array().unwrap().len(), 3);
+ assert!(
+ models["data"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .all(|model| model["context_length"].is_null())
+ );
+ assert_eq!(send(&app, "GET", "/v1/models", None).await?.json()?, models);
+ assert_eq!(upstream.calls.lock().await.len(), 3);
+ }
+ Ok(())
+}
+
#[tokio::test]
async fn vgr_serves_verified_attempts_locally_and_escalates_rejected_ones() -> TestResult {
let upstream = MockUpstream::start().await?;
- for (judge, served) in [
- ("model/vgr-judge", "model/vgr-local"),
- ("model/vgr-reject", "model/vgr-cloud"),
+ for (discovery, configured, expected_context, probes) in [
+ ("", "context_window = 32768", Some(32_768), false),
+ ("", "", None, false),
+ ("context_discovery = \"llama_cpp\"", "", Some(98_304), true),
+ (
+ "context_discovery = \"llama_cpp\"",
+ "context_window = 32768",
+ Some(32_768),
+ false,
+ ),
] {
- let app = build_switchyard_router(load_test_config(&format!(
- r#"
+ for (judge, served) in [
+ ("model/vgr-judge", "model/vgr-local"),
+ ("model/vgr-reject", "model/vgr-cloud"),
+ ] {
+ upstream.calls.lock().await.clear();
+ let app = build_switchyard_router(load_test_config(&format!(
+ r#"
schema_version = 1
[llm_clients.primary]
format = "openai_chat"
base_url = "{}"
+{discovery}
[targets.local]
id = "model/vgr-local"
@@ -1713,7 +1810,7 @@ llm_client = "primary"
[routes.vgr]
id = "switchyard/vgr"
type = "vgr"
-context_window = 32768
+{configured}
local_target = "local"
cloud_target = "cloud"
judge_target = "judge"
@@ -1721,33 +1818,37 @@ mode = "active"
active_approval = "prospective-validation-and-canary-approved"
task_typing = false
"#,
- upstream.base_url
- ))?);
- let response = send(
- &app,
- "POST",
- "/v1/chat/completions",
- Some(json!({
- "model": "switchyard/vgr",
- "messages": [{"role": "user", "content": "answer the question"}]
- })),
- )
- .await?;
- assert_eq!(response.status, StatusCode::OK);
- assert_eq!(
- response.headers.get("x-model-router-selected-model"),
- Some(&HeaderValue::from_str(served)?)
- );
- let models = send(&app, "GET", "/v1/models", None).await?.json()?;
- assert_eq!(models["data"][0]["context_length"], 32_768);
- assert!(
- upstream
- .calls
- .lock()
- .await
- .iter()
- .all(|call| call["path"] != "/props")
- );
+ upstream.base_url
+ ))?);
+ let response = send(
+ &app,
+ "POST",
+ "/v1/chat/completions",
+ Some(json!({
+ "model": "switchyard/vgr",
+ "messages": [{"role": "user", "content": "answer the question"}]
+ })),
+ )
+ .await?;
+ assert_eq!(response.status, StatusCode::OK);
+ assert_eq!(
+ response.headers.get("x-model-router-selected-model"),
+ Some(&HeaderValue::from_str(served)?)
+ );
+ let models = send(&app, "GET", "/v1/models", None).await?.json()?;
+ assert_eq!(models["data"][0]["context_length"], json!(expected_context));
+ assert_eq!(send(&app, "GET", "/v1/models", None).await?.json()?, models);
+ assert_eq!(
+ upstream
+ .calls
+ .lock()
+ .await
+ .iter()
+ .filter(|call| call["path"] == "/props")
+ .count(),
+ usize::from(probes)
+ );
+ }
}
Ok(())
}
diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md
index d17259916..b3160a4c7 100644
--- a/docs/reference/toml_schema.md
+++ b/docs/reference/toml_schema.md
@@ -50,6 +50,7 @@ route reaches no upstream. A file without a `[targets]` table is rejected with
|---|:---:|---|---|
| `format` | Yes | — | `openai_chat`, `openai_responses`, or `anthropic_messages`. |
| `base_url` | Yes | — | Upstream base URL. |
+| `context_discovery` | No | unset | Set to `"llama_cpp"` to enable `/props` context discovery for VGR local targets. Disabled when unset. An explicit route `context_window` takes precedence. |
| `api_key_env` | No | unset | Name of the environment variable holding the key. Omit to send no authentication. |
| `forward_auth` | No | `false` | Forward the caller's provider credential and application headers. A route's forwarding clients must use one credential family unless they all use the same scheme, host, and port, such as one LLM gateway. |
| `extra_headers` | No | `{}` | Custom HTTP headers sent to the model server. Set credentials with `api_key_env` or `forward_auth`; the server rejects headers owned by the selected auth mode. Header names are case-insensitive. |
diff --git a/docs/routing_algorithms/vgr_routing.md b/docs/routing_algorithms/vgr_routing.md
index 21cc9d4f9..9378703a9 100644
--- a/docs/routing_algorithms/vgr_routing.md
+++ b/docs/routing_algorithms/vgr_routing.md
@@ -32,8 +32,26 @@ confirmed_recovery_min_clean_tail = 1
```
`judge_target` defaults to `local_target`. `cloud_judge_target` is optional.
-The local and cloud targets must resolve to distinct model IDs. When the
-route sets `context_window`, `/v1/models` advertises that configured limit.
+The local and cloud targets must resolve to distinct model IDs. Set the route's
+`context_window` to advertise a configured limit through `/v1/models`; this works
+with any supported local service and takes precedence over discovery.
+
+Context discovery is disabled by default. For llama.cpp, set
+`context_discovery = "llama_cpp"` on the local target's `[llm_clients.<name>]`
+entry and leave the route's `context_window` unset. Switchyard then reads the
+service's `/props` response and advertises a positive context capacity when
+available. Missing or invalid properties leave the capacity unknown; routing
+continues normally. Discovery does not configure the service or load a model.
+
+See [the local-server example](../../examples/vgr-local-servers.toml)
+for an active llama.cpp configuration and commented Ollama/vLLM alternatives.
+For Ollama and vLLM, set `context_window` to match the running service; automatic
+discovery for those servers is not implemented.
+
+The server caches the model catalog, including unavailable discovery results,
+for one minute. Discovery probes run concurrently with a two-second total budget
+per refresh. Completed discoveries are retained when that budget expires.
+
`confirmed_recovery_min_clean_tail`, when set, lets an agentic
run that recovered from tool errors commit locally once that many trailing tool
results are clean and the cloud judge confirms the evidence.
diff --git a/examples/vgr-local-servers.toml b/examples/vgr-local-servers.toml
new file mode 100644
index 000000000..dfa789cf7
--- /dev/null
+++ b/examples/vgr-local-servers.toml
@@ -0,0 +1,48 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Replace model IDs, addresses, and cloud authentication for your deployment.
+schema_version = 1
+
+# llama.cpp is active in this example. Discovery must be explicitly enabled.
+[llm_clients.local]
+format = "openai_chat"
+base_url = "http://localhost:8080/v1"
+context_discovery = "llama_cpp"
+
+# For Ollama or vLLM, replace the active local client block with ONE block below.
+# Also set context_window in routes.assistant to match the running service.
+#
+# Ollama:
+# [llm_clients.local]
+# format = "openai_chat"
+# base_url = "http://localhost:11434/v1"
+#
+# vLLM:
+# [llm_clients.local]
+# format = "openai_chat"
+# base_url = "http://localhost:8000/v1"
+
+[llm_clients.cloud]
+format = "openai_chat"
+base_url = "https://api.openai.com/v1"
+# api_key_env = "OPENAI_API_KEY"
+
+[targets.local]
+# Use the llama.cpp alias, Ollama model tag, or vLLM served model name.
+id = "local-model"
+llm_client = "local"
+
+[targets.cloud]
+id = "cloud-model"
+llm_client = "cloud"
+
+[routes.assistant]
+id = "switchyard/vgr"
+type = "vgr"
+local_target = "local"
+cloud_target = "cloud"
+mode = "shadow"
+# Explicit context_window takes precedence and skips discovery.
+# Set this for Ollama/vLLM, or to set a deliberate limit with llama.cpp.
+# context_window = 32768