Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(vgr): add optional execution limits and Hermes auxiliary example (#1171)

GDRSeabeac0883bc
diff --git a/crates/libsy/src/algorithms/vgr/config.rs b/crates/libsy/src/algorithms/vgr/config.rs
index c390e2102..1c730a10d 100644
--- a/crates/libsy/src/algorithms/vgr/config.rs
+++ b/crates/libsy/src/algorithms/vgr/config.rs
@@ -50,6 +50,8 @@ pub struct VgrConfig {
pub mode: ServingMode,
/// Budget for one turn's attempt and verification.
pub deadline: Duration,
+ /// Enforces the decision deadline and efficient turn budget. Defaults to true.
+ pub bounded_efficient: bool,
/// Optional live stop controlled by the operator.
pub kill_switch: Option<KillSwitch>,
/// Efficient endpoint breaker tuning.
@@ -86,6 +88,7 @@ impl VgrConfig {
},
mode: ServingMode::Off,
deadline: Duration::from_secs(30),
+ bounded_efficient: true,
kill_switch: None,
breaker: BreakerConfig::default(),
task_typing: true,
diff --git a/crates/libsy/src/algorithms/vgr/runtime.rs b/crates/libsy/src/algorithms/vgr/runtime.rs
index a70c4acb8..6d8ac7f3c 100644
--- a/crates/libsy/src/algorithms/vgr/runtime.rs
+++ b/crates/libsy/src/algorithms/vgr/runtime.rs
@@ -5,10 +5,10 @@
//!
//! A proposed tool call is judged against the session's trajectory; a final
//! answer is typed, judged by rungs that run cheapest-first and stop once the
-//! decision is settled, and then committed or escalated. Every call is bounded
-//! by what is left of the turn's deadline, and a call that fails or times out
-//! is evidence never gathered, which does not commit.
+//! decision is settled, and then committed or escalated. Bounded mode shares one
+//! deadline across calls. A failed or timed-out call supplies no evidence.
+use std::future::Future;
use std::sync::Arc;
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
@@ -165,25 +165,26 @@ impl Classifier for VgrClassifier {
return Ok(self.escalate(driver, reason, None, request, state, None));
}
- let Some(budget) = self.remaining(started) else {
+ if self.config.bounded_efficient && self.remaining(started).is_none() {
return Ok(self.escalate(driver, "efficient_timed_out", None, request, state, None));
- };
+ }
let Some(trial) = self.breaker.attempt() else {
return Ok(self.escalate(driver, "breaker_open", None, request, state, None));
};
- let attempted = tokio::time::timeout(budget, async {
- let response = driver
- .call_model_with_error_recovery(request.clone(), vec![efficient.clone()], true)
- .await?;
- buffer_response(efficient.as_str(), response).await
- })
- .await;
+ let attempted = self
+ .within_budget(started, async {
+ let response = driver
+ .call_model_with_error_recovery(request.clone(), vec![efficient.clone()], true)
+ .await?;
+ buffer_response(efficient.as_str(), response).await
+ })
+ .await;
let buffered = match attempted {
- Ok(Ok(buffered)) => {
+ Some(Ok(buffered)) => {
trial.success();
buffered
}
- Ok(Err(error)) => {
+ Some(Err(error)) => {
if endpoint_failure(&error) {
trial.failure();
}
@@ -199,7 +200,7 @@ impl Classifier for VgrClassifier {
None,
));
}
- Err(_) => {
+ None => {
trial.failure();
return Ok(self.escalate(
driver,
@@ -349,6 +350,9 @@ impl VgrClassifier {
/// client's tool execution, since that is the time the client's own limit
/// is spending.
fn turn_budget_spent(&self, state: &mut State) -> bool {
+ if !self.config.bounded_efficient {
+ return false;
+ }
let Some(budget) = self.config.efficient_turn_budget else {
return false;
};
@@ -565,8 +569,7 @@ impl VgrClassifier {
call: Request,
target: &ModelId,
) -> Option<Result<AggLlmResponse>> {
- let budget = self.remaining(context.started)?;
- let outcome = tokio::time::timeout(budget, async {
+ self.within_budget(context.started, async {
let response = context
.driver
.call_model_with_error_recovery(call, vec![target.clone()], true)
@@ -577,8 +580,21 @@ impl VgrClassifier {
.await
.map_err(|source| LibsyError::client_call(target.clone(), source))
})
- .await;
- outcome.ok()
+ .await
+ }
+
+ /// Waits for a candidate or verifier, enforcing a shared deadline only in bounded mode.
+ async fn within_budget<T>(
+ &self,
+ started: Instant,
+ operation: impl Future<Output = T>,
+ ) -> Option<T> {
+ if !self.config.bounded_efficient {
+ return Some(operation.await);
+ }
+ tokio::time::timeout(self.remaining(started)?, operation)
+ .await
+ .ok()
}
fn remaining(&self, started: Instant) -> Option<Duration> {
diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rs
index b7036bed5..c7d0882dc 100644
--- a/crates/switchyard-runner/src/algorithm.rs
+++ b/crates/switchyard-runner/src/algorithm.rs
@@ -540,6 +540,9 @@ pub struct VgrRouteConfig {
pub active_approval: Option<String>,
#[serde(default = "default_vgr_deadline")]
pub deadline_seconds: f64,
+ /// Enforces VGR decision and efficient turn budgets. Defaults to true.
+ #[serde(default = "default_true")]
+ pub bounded_efficient: bool,
#[serde(default = "default_true")]
pub task_typing: bool,
#[serde(default = "default_vgr_breaker_threshold")]
@@ -1844,6 +1847,7 @@ fn build_vgr(
},
};
runtime.deadline = vgr_duration(route_name, "deadline_seconds", config.deadline_seconds)?;
+ runtime.bounded_efficient = config.bounded_efficient;
runtime.task_typing = config.task_typing;
runtime.confirmed_recovery_min_clean_tail = config.confirmed_recovery_min_clean_tail;
runtime.agentic_handoff = config.agentic_handoff;
diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs
index 406247ca9..c01793053 100644
--- a/crates/switchyard-runner/src/config.rs
+++ b/crates/switchyard-runner/src/config.rs
@@ -1365,6 +1365,12 @@ task_typing = false
let example = include_str!("../../../examples/vgr-local-servers.toml");
let runner = runner_from_toml(example)?;
assert!(runner.route("switchyard/vgr").is_some());
+ assert!(runner.route("switchyard/only-efficient").is_some());
+ let parsed: DeploymentConfig = toml::from_str(example).expect("valid VGR example");
+ assert!(matches!(
+ &parsed.routes["auxiliary"].algorithm,
+ AlgorithmSpec::Passthrough { target, .. } if target == "efficient"
+ ));
let invalid = example.replace(
"context_discovery = \"llama_cpp\"",
"context_discovery = \"guess\"",
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index 73062b251..51299552f 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -6028,3 +6028,55 @@ seed = 17
}
Ok(())
}
+
+#[tokio::test]
+async fn vgr_time_limits_can_be_disabled_from_toml() -> TestResult {
+ let upstream = MockUpstream::start().await?;
+ for limit in ["deadline_seconds", "efficient_turn_budget_seconds"] {
+ for (configured, served) in [
+ ("", "model/vgr-capable"),
+ ("bounded_efficient = false", "model/vgr-efficient"),
+ ] {
+ let app = build_switchyard_router(load_test_config(&format!(
+ r#"
+schema_version = 1
+[llm_clients.primary]
+format = "openai_chat"
+base_url = "{}"
+[targets]
+efficient = {{ id = "model/vgr-efficient", llm_client = "primary" }}
+capable = {{ id = "model/vgr-capable", llm_client = "primary" }}
+judge = {{ id = "model/vgr-judge", llm_client = "primary" }}
+[routes.vgr]
+id = "switchyard/vgr"
+type = "vgr"
+efficient_target = "efficient"
+capable_target = "capable"
+judge_target = "judge"
+mode = "evaluate"
+task_typing = false
+{configured}
+{limit} = 0.000000001
+"#,
+ upstream.base_url
+ ))?);
+ let response = send(
+ &app,
+ "POST",
+ "/v1/chat/completions",
+ Some(json!({
+ "model": "switchyard/vgr",
+ "messages": [{"role": "user", "content": "answer the question"}]
+ })),
+ )
+ .await?;
+ assert_eq!(response.status, StatusCode::OK, "{}", response.text()?);
+ assert_eq!(
+ response.headers.get("x-model-router-selected-model"),
+ Some(&HeaderValue::from_str(served)?),
+ "{limit}, {configured}"
+ );
+ }
+ }
+ Ok(())
+}
diff --git a/docs/routing_algorithms/vgr_routing.md b/docs/routing_algorithms/vgr_routing.md
index 64d856c42..d8140bc4d 100644
--- a/docs/routing_algorithms/vgr_routing.md
+++ b/docs/routing_algorithms/vgr_routing.md
@@ -110,6 +110,42 @@ Escalations release when a new user turn is decided on the efficient tier. Set
Shadow mode always releases the hold at the next user turn.
All configured VGR durations must be positive.
+## Execution limits
+
+`bounded_efficient` defaults to `true`. Set `bounded_efficient = false` on the
+VGR route to disable its decision deadline and `efficient_turn_budget_seconds`.
+Candidate generation, task typing, and verification then wait for completion.
+Client and upstream HTTP timeouts still apply. Configure those timeouts in the
+harness and Switchyard client for long-running decisions. Configured durations
+must remain positive even when these VGR limits are disabled.
+
+## Hermes auxiliary routing
+
+The [local-server example](../../examples/vgr-local-servers.toml) exposes
+`switchyard/only-efficient` through the existing passthrough router. Hermes can
+use it for titles and approval checks, independently of the conversation's VGR
+route. Configure Hermes against the same Switchyard server:
+
+```yaml
+auxiliary:
+ title_generation:
+ provider: custom
+ model: switchyard/only-efficient
+ base_url: http://localhost:4000/v1
+ approval:
+ provider: custom
+ model: switchyard/only-efficient
+ base_url: http://localhost:4000/v1
+ extra_body:
+ reasoning_effort: none
+ chat_template_kwargs:
+ enable_thinking: false
+```
+
+Use the approval thinking controls only with endpoints that support them.
+Adjust auxiliary timeouts for endpoint queueing and allow concurrent requests
+on the efficient server when titles must run alongside the conversation.
+
## Hermes session headers
Use a Hermes build that supports both provider options below. The session header
diff --git a/examples/vgr-local-servers.toml b/examples/vgr-local-servers.toml
index 82f9c6093..783e2b0f3 100644
--- a/examples/vgr-local-servers.toml
+++ b/examples/vgr-local-servers.toml
@@ -46,3 +46,9 @@ mode = "shadow"
# Explicit context_window takes precedence and skips discovery.
# Set this for Ollama/vLLM, or to set a deliberate limit with llama.cpp.
# context_window = 32768
+
+# Hermes titles and approval checks bypass VGR verification.
+[routes.auxiliary]
+id = "switchyard/only-efficient"
+type = "passthrough"
+target = "efficient"