Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(vgr): add optional execution limits and Hermes auxiliary example (#1171)
GDRSeabeac0883bc
diff --git a/crates/libsy/src/algorithms/vgr/config.rs b/crates/libsy/src/algorithms/vgr/config.rsindex c390e2102..1c730a10d 100644--- a/crates/libsy/src/algorithms/vgr/config.rs+++ b/crates/libsy/src/algorithms/vgr/config.rs@@ -50,6 +50,8 @@ pub struct VgrConfig {pub mode: ServingMode,/// Budget for one turn's attempt and verification.pub deadline: Duration,+ /// Enforces the decision deadline and efficient turn budget. Defaults to true.+ pub bounded_efficient: bool,/// Optional live stop controlled by the operator.pub kill_switch: Option<KillSwitch>,/// Efficient endpoint breaker tuning.@@ -86,6 +88,7 @@ impl VgrConfig {},mode: ServingMode::Off,deadline: Duration::from_secs(30),+ bounded_efficient: true,kill_switch: None,breaker: BreakerConfig::default(),task_typing: true,diff --git a/crates/libsy/src/algorithms/vgr/runtime.rs b/crates/libsy/src/algorithms/vgr/runtime.rsindex a70c4acb8..6d8ac7f3c 100644--- a/crates/libsy/src/algorithms/vgr/runtime.rs+++ b/crates/libsy/src/algorithms/vgr/runtime.rs@@ -5,10 +5,10 @@//!//! A proposed tool call is judged against the session's trajectory; a final//! answer is typed, judged by rungs that run cheapest-first and stop once the-//! decision is settled, and then committed or escalated. Every call is bounded-//! by what is left of the turn's deadline, and a call that fails or times out-//! is evidence never gathered, which does not commit.+//! decision is settled, and then committed or escalated. Bounded mode shares one+//! deadline across calls. A failed or timed-out call supplies no evidence.+use std::future::Future;use std::sync::Arc;use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};@@ -165,25 +165,26 @@ impl Classifier for VgrClassifier {return Ok(self.escalate(driver, reason, None, request, state, None));}- let Some(budget) = self.remaining(started) else {+ if self.config.bounded_efficient && self.remaining(started).is_none() {return Ok(self.escalate(driver, "efficient_timed_out", None, request, state, None));- };+ }let Some(trial) = self.breaker.attempt() else {return Ok(self.escalate(driver, "breaker_open", None, request, state, None));};- let attempted = tokio::time::timeout(budget, async {- let response = driver- .call_model_with_error_recovery(request.clone(), vec![efficient.clone()], true)- .await?;- buffer_response(efficient.as_str(), response).await- })- .await;+ let attempted = self+ .within_budget(started, async {+ let response = driver+ .call_model_with_error_recovery(request.clone(), vec![efficient.clone()], true)+ .await?;+ buffer_response(efficient.as_str(), response).await+ })+ .await;let buffered = match attempted {- Ok(Ok(buffered)) => {+ Some(Ok(buffered)) => {trial.success();buffered}- Ok(Err(error)) => {+ Some(Err(error)) => {if endpoint_failure(&error) {trial.failure();}@@ -199,7 +200,7 @@ impl Classifier for VgrClassifier {None,));}- Err(_) => {+ None => {trial.failure();return Ok(self.escalate(driver,@@ -349,6 +350,9 @@ impl VgrClassifier {/// client's tool execution, since that is the time the client's own limit/// is spending.fn turn_budget_spent(&self, state: &mut State) -> bool {+ if !self.config.bounded_efficient {+ return false;+ }let Some(budget) = self.config.efficient_turn_budget else {return false;};@@ -565,8 +569,7 @@ impl VgrClassifier {call: Request,target: &ModelId,) -> Option<Result<AggLlmResponse>> {- let budget = self.remaining(context.started)?;- let outcome = tokio::time::timeout(budget, async {+ self.within_budget(context.started, async {let response = context.driver.call_model_with_error_recovery(call, vec![target.clone()], true)@@ -577,8 +580,21 @@ impl VgrClassifier {.await.map_err(|source| LibsyError::client_call(target.clone(), source))})- .await;- outcome.ok()+ .await+ }++ /// Waits for a candidate or verifier, enforcing a shared deadline only in bounded mode.+ async fn within_budget<T>(+ &self,+ started: Instant,+ operation: impl Future<Output = T>,+ ) -> Option<T> {+ if !self.config.bounded_efficient {+ return Some(operation.await);+ }+ tokio::time::timeout(self.remaining(started)?, operation)+ .await+ .ok()}fn remaining(&self, started: Instant) -> Option<Duration> {diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rsindex b7036bed5..c7d0882dc 100644--- a/crates/switchyard-runner/src/algorithm.rs+++ b/crates/switchyard-runner/src/algorithm.rs@@ -540,6 +540,9 @@ pub struct VgrRouteConfig {pub active_approval: Option<String>,#[serde(default = "default_vgr_deadline")]pub deadline_seconds: f64,+ /// Enforces VGR decision and efficient turn budgets. Defaults to true.+ #[serde(default = "default_true")]+ pub bounded_efficient: bool,#[serde(default = "default_true")]pub task_typing: bool,#[serde(default = "default_vgr_breaker_threshold")]@@ -1844,6 +1847,7 @@ fn build_vgr(},};runtime.deadline = vgr_duration(route_name, "deadline_seconds", config.deadline_seconds)?;+ runtime.bounded_efficient = config.bounded_efficient;runtime.task_typing = config.task_typing;runtime.confirmed_recovery_min_clean_tail = config.confirmed_recovery_min_clean_tail;runtime.agentic_handoff = config.agentic_handoff;diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rsindex 406247ca9..c01793053 100644--- a/crates/switchyard-runner/src/config.rs+++ b/crates/switchyard-runner/src/config.rs@@ -1365,6 +1365,12 @@ task_typing = falselet example = include_str!("../../../examples/vgr-local-servers.toml");let runner = runner_from_toml(example)?;assert!(runner.route("switchyard/vgr").is_some());+ assert!(runner.route("switchyard/only-efficient").is_some());+ let parsed: DeploymentConfig = toml::from_str(example).expect("valid VGR example");+ assert!(matches!(+ &parsed.routes["auxiliary"].algorithm,+ AlgorithmSpec::Passthrough { target, .. } if target == "efficient"+ ));let invalid = example.replace("context_discovery = \"llama_cpp\"","context_discovery = \"guess\"",diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rsindex 73062b251..51299552f 100644--- a/crates/switchyard-server/tests/server.rs+++ b/crates/switchyard-server/tests/server.rs@@ -6028,3 +6028,55 @@ seed = 17}Ok(())}++#[tokio::test]+async fn vgr_time_limits_can_be_disabled_from_toml() -> TestResult {+ let upstream = MockUpstream::start().await?;+ for limit in ["deadline_seconds", "efficient_turn_budget_seconds"] {+ for (configured, served) in [+ ("", "model/vgr-capable"),+ ("bounded_efficient = false", "model/vgr-efficient"),+ ] {+ let app = build_switchyard_router(load_test_config(&format!(+ r#"+schema_version = 1+[llm_clients.primary]+format = "openai_chat"+base_url = "{}"+[targets]+efficient = {{ id = "model/vgr-efficient", llm_client = "primary" }}+capable = {{ id = "model/vgr-capable", llm_client = "primary" }}+judge = {{ id = "model/vgr-judge", llm_client = "primary" }}+[routes.vgr]+id = "switchyard/vgr"+type = "vgr"+efficient_target = "efficient"+capable_target = "capable"+judge_target = "judge"+mode = "evaluate"+task_typing = false+{configured}+{limit} = 0.000000001+"#,+ upstream.base_url+ ))?);+ let response = send(+ &app,+ "POST",+ "/v1/chat/completions",+ Some(json!({+ "model": "switchyard/vgr",+ "messages": [{"role": "user", "content": "answer the question"}]+ })),+ )+ .await?;+ assert_eq!(response.status, StatusCode::OK, "{}", response.text()?);+ assert_eq!(+ response.headers.get("x-model-router-selected-model"),+ Some(&HeaderValue::from_str(served)?),+ "{limit}, {configured}"+ );+ }+ }+ Ok(())+}diff --git a/docs/routing_algorithms/vgr_routing.md b/docs/routing_algorithms/vgr_routing.mdindex 64d856c42..d8140bc4d 100644--- a/docs/routing_algorithms/vgr_routing.md+++ b/docs/routing_algorithms/vgr_routing.md@@ -110,6 +110,42 @@ Escalations release when a new user turn is decided on the efficient tier. SetShadow mode always releases the hold at the next user turn.All configured VGR durations must be positive.+## Execution limits++`bounded_efficient` defaults to `true`. Set `bounded_efficient = false` on the+VGR route to disable its decision deadline and `efficient_turn_budget_seconds`.+Candidate generation, task typing, and verification then wait for completion.+Client and upstream HTTP timeouts still apply. Configure those timeouts in the+harness and Switchyard client for long-running decisions. Configured durations+must remain positive even when these VGR limits are disabled.++## Hermes auxiliary routing++The [local-server example](../../examples/vgr-local-servers.toml) exposes+`switchyard/only-efficient` through the existing passthrough router. Hermes can+use it for titles and approval checks, independently of the conversation's VGR+route. Configure Hermes against the same Switchyard server:++```yaml+auxiliary:+ title_generation:+ provider: custom+ model: switchyard/only-efficient+ base_url: http://localhost:4000/v1+ approval:+ provider: custom+ model: switchyard/only-efficient+ base_url: http://localhost:4000/v1+ extra_body:+ reasoning_effort: none+ chat_template_kwargs:+ enable_thinking: false+```++Use the approval thinking controls only with endpoints that support them.+Adjust auxiliary timeouts for endpoint queueing and allow concurrent requests+on the efficient server when titles must run alongside the conversation.+## Hermes session headersUse a Hermes build that supports both provider options below. The session headerdiff --git a/examples/vgr-local-servers.toml b/examples/vgr-local-servers.tomlindex 82f9c6093..783e2b0f3 100644--- a/examples/vgr-local-servers.toml+++ b/examples/vgr-local-servers.toml@@ -46,3 +46,9 @@ mode = "shadow"# Explicit context_window takes precedence and skips discovery.# Set this for Ollama/vLLM, or to set a deliberate limit with llama.cpp.# context_window = 32768++# Hermes titles and approval checks bypass VGR verification.+[routes.auxiliary]+id = "switchyard/only-efficient"+type = "passthrough"+target = "efficient"