Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
49.9 KB
Raw Blame
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};

use serde_json::json;
use switchyard_protocol::{
    AggLlmResponse, Category, ContentBlock, FormatId, LlmClientError, LlmResponse, Message,
    Metadata, ModelId, Request, Response, ResponseOutput, Role, StopReason, ToolCall, WireFormat,
};

use super::super::config::{ACTIVE_APPROVAL, ServingMode, VgrConfig};
use super::super::tests::{call, request, result};
use super::{FOLLOW_UP_QUIET_SECS, LAST_ANSWER_FINAL_KEY, answers_final_reply, unix_seconds};
use crate::Result;
use crate::algorithms::vgr::Vgr;
use crate::core::algorithm::{Algorithm, AnswerSummary};
use crate::core::state::{State, StateValue};
use crate::core::testing::{category_models, reply, test_drive_with_models};

fn route(configure: impl FnOnce(&mut VgrConfig)) -> Result<Arc<dyn Algorithm>> {
    let mut config = VgrConfig::new(ModelId::new("efficient"), ModelId::new("capable"));
    config.targets.judge = Some(ModelId::new("judge"));
    config.targets.capable_judge = Some(ModelId::new("capable-judge"));
    config.mode = ServingMode::Active {
        approval: ACTIVE_APPROVAL.into(),
    };
    config.task_typing = false;
    configure(&mut config);
    Ok(Arc::new(Vgr::new(config)?))
}

fn session(messages: Vec<Message>) -> Request {
    let mut request = request(messages);
    request.metadata = Some(Metadata {
        session_id: Some("session-1".into()),
        ..Metadata::default()
    });
    request
}

fn agg(response: AggLlmResponse) -> Response {
    Response {
        llm_response: LlmResponse::Agg(response),
        metadata: None,
        upstream_headers: http::HeaderMap::new(),
    }
}

fn tool_call(arguments: serde_json::Value, stop_reason: StopReason) -> Response {
    agg(AggLlmResponse {
        outputs: vec![ResponseOutput {
            role: Role::Assistant,
            content: vec![ContentBlock::ToolCall(ToolCall {
                id: "next".into(),
                name: "bash".into(),
                arguments,
            })],
            url_citations: Vec::new(),
            stop_reason: Some(stop_reason),
        }],
        ..Default::default()
    })
}

/// A verifier reply whose first token puts probability `p` on "yes".
fn scored(p: f64) -> Response {
    let mut response = AggLlmResponse::default();
    response.preservation.responses.insert(
        FormatId::from(WireFormat::OpenAiChat),
        json!({"choices": [{"logprobs": {"content": [{"top_logprobs": [
            {"token": "yes", "logprob": p.ln()},
            {"token": "no", "logprob": (1.0 - p).ln()}
        ]}]}}]}),
    );
    agg(response)
}

#[derive(Default)]
struct Calls(AtomicUsize, AtomicUsize);

async fn drive(
    route: &Arc<dyn Algorithm>,
    request: Request,
    calls: Arc<Calls>,
    efficient: fn() -> Response,
    judge_p: f64,
) -> Result<String> {
    let (selected, _) = test_drive_with_models(
        Arc::clone(route),
        request,
        category_models(Category::Any, &["efficient", "capable"]),
        move |target: ModelId, _| {
            let calls = Arc::clone(&calls);
            async move {
                match target.as_str() {
                    "efficient" => calls.0.fetch_add(1, Ordering::Relaxed),
                    "capable-judge" => calls.1.fetch_add(1, Ordering::Relaxed),
                    _ => 0,
                };
                Ok(match target.as_str() {
                    "efficient" => efficient(),
                    "judge" => scored(judge_p),
                    "capable-judge" => reply("yes"),
                    _ => reply("capable answer"),
                })
            }
        },
    )
    .await?;
    Ok(selected.to_string())
}

fn complete_call() -> Response {
    tool_call(json!({"command": "ls"}), StopReason::ToolUse)
}

#[tokio::test]
async fn malformed_tool_calls_escalate_and_complete_ones_are_judged() -> Result<()> {
    let route = route(|_| {})?;
    let calls = Arc::new(Calls::default());
    let truncated = || tool_call(json!(r#"{"command":"l"#), StopReason::MaxTokens);
    let task = || session(vec![Message::text(Role::User, "list files")]);
    assert_eq!(
        drive(&route, task(), calls.clone(), truncated, 0.0).await?,
        "capable"
    );
    assert_eq!(
        drive(&route, task(), calls, complete_call, 0.1).await?,
        "efficient"
    );
    Ok(())
}

#[tokio::test]
async fn a_truncated_efficient_attempt_is_judged_before_it_escalates() -> Result<()> {
    let route = route(|config| {
        config.targets.judge = None;
        config.targets.capable_judge = None;
    })?;
    let calls = Arc::new(Calls::default());
    let task = session(vec![Message::text(Role::User, "explain quicksort")]);
    let cut_off = || {
        let mut response = reply("Quicksort picks a pivot, then recurses because");
        if let LlmResponse::Agg(agg) = &mut response.llm_response {
            agg.outputs[0].stop_reason = Some(StopReason::MaxTokens);
        }
        response
    };
    let selected = drive(&route, task, calls.clone(), cut_off, 0.0).await?;
    assert_eq!(selected, "capable");
    // Attempt, readout, deliberation and its verdict; read as absent, only the attempt runs.
    assert_eq!(calls.0.load(Ordering::Relaxed), 4);
    Ok(())
}

#[tokio::test]
async fn two_consecutive_escalation_votes_latch_the_session() -> Result<()> {
    let route = route(|_| {})?;
    let calls = Arc::new(Calls::default());
    let task = || session(vec![Message::text(Role::User, "fix the build")]);
    assert_eq!(
        drive(&route, task(), calls.clone(), complete_call, 0.9).await?,
        "efficient"
    );
    assert_eq!(
        drive(&route, task(), calls.clone(), complete_call, 0.9).await?,
        "capable"
    );
    assert_eq!(calls.0.load(Ordering::Relaxed), 2);

    // Latched past the user turn: no further efficient attempt is produced.
    assert_eq!(
        drive(&route, task(), calls.clone(), complete_call, 0.0).await?,
        "capable"
    );
    assert_eq!(calls.0.load(Ordering::Relaxed), 2);
    Ok(())
}

#[tokio::test]
async fn final_answers_commit_only_on_a_verified_clean_record() -> Result<()> {
    let answer = || reply("the build passes");
    let run = |tool_output: &str| {
        session(vec![
            Message::text(Role::User, "fix the build"),
            call("c1"),
            result("c1", tool_output),
        ])
    };
    let calls = Arc::new(Calls::default());
    let clean = route(|_| {})?;
    assert_eq!(
        drive(&clean, run("ok"), calls.clone(), answer, 0.3).await?,
        "efficient"
    );
    assert_eq!(
        drive(&clean, run("ok"), calls.clone(), answer, 0.1).await?,
        "capable"
    );

    // A failure longer ago than the clean tail is forgiven only on the capable judge.
    let mut long = vec![Message::text(Role::User, "fix the build")];
    long.extend([call("e"), result("e", r#"{"error": "missing file"}"#)]);
    for index in 0..101 {
        let id = format!("c{index}");
        long.extend([call(&id), result(&id, "ok")]);
    }
    assert_eq!(
        drive(&clean, session(long.clone()), calls.clone(), answer, 0.99).await?,
        "capable"
    );
    assert_eq!(calls.1.load(Ordering::Relaxed), 0);
    let confirmed = route(|config| config.confirmed_recovery_min_clean_tail = Some(1))?;
    assert_eq!(
        drive(&confirmed, session(long), calls.clone(), answer, 0.99).await?,
        "efficient"
    );
    assert_eq!(calls.1.load(Ordering::Relaxed), 1);
    Ok(())
}

#[tokio::test]
async fn a_typed_follow_up_answer_is_judged_on_the_answer_itself() -> Result<()> {
    let route = route(|config| config.task_typing = true)?;
    let conversation = session(vec![
        Message::text(Role::User, "fix the build"),
        call("c1"),
        result("c1", "ok"),
        Message::text(Role::Assistant, "The build passes."),
        Message::text(Role::User, "What is 1+1?"),
    ]);
    // A confident readout serves on the efficient tier with no capable-tier call; a weak one asks the
    // capable tier's answer verifier, with no efficient deliberation first.
    for (readout, expected_calls) in [
        (0.95, vec!["judge: typing", "judge: answer"]),
        (
            0.5,
            vec!["judge: typing", "judge: answer", "capable-judge: answer"],
        ),
    ] {
        let calls = Arc::new(parking_lot::Mutex::new(Vec::new()));
        let seen = Arc::clone(&calls);
        let (selected, _) = test_drive_with_models(
            Arc::clone(&route),
            conversation.clone(),
            category_models(Category::Any, &["efficient", "capable"]),
            move |target: ModelId, request: Request| {
                let seen = Arc::clone(&seen);
                async move {
                    let system = request
                        .llm_request
                        .instructions
                        .first()
                        .and_then(|block| block.content.first())
                        .and_then(|block| match block {
                            ContentBlock::Text { text } => Some(text.clone()),
                            _ => None,
                        })
                        .unwrap_or_default();
                    let question = if system.starts_with("You are a strict answer verifier") {
                        "answer"
                    } else if request
                        .llm_request
                        .extensions
                        .fields
                        .contains_key("logprobs")
                    {
                        "evidence"
                    } else {
                        "typing"
                    };
                    if target.as_str().contains("judge") {
                        seen.lock().push(format!("{}: {question}", target.as_str()));
                    }
                    Ok(match target.as_str() {
                        "efficient" => reply("2"),
                        "judge" if question == "typing" => reply("Answer"),
                        "judge" => scored(readout),
                        "capable-judge" => reply("yes"),
                        _ => reply("capable answer"),
                    })
                }
            },
        )
        .await?;
        assert_eq!(selected.as_str(), "efficient", "readout {readout}");
        assert_eq!(*calls.lock(), expected_calls, "readout {readout}");
    }
    Ok(())
}

#[tokio::test]
async fn an_unavailable_efficient_tier_escalates_and_other_failures_surface() -> Result<()> {
    let route = route(|_| {})?;
    for (status, escalates) in [(503, true), (400, false)] {
        let outcome = test_drive_with_models(
            Arc::clone(&route),
            session(vec![Message::text(Role::User, "task")]),
            category_models(Category::Any, &["efficient", "capable"]),
            move |target: ModelId, _| async move {
                match target.as_str() {
                    "efficient" => Err(LlmClientError::UpstreamHttp {
                        status: http::StatusCode::from_u16(status).unwrap_or_default(),
                        body: String::new(),
                    }),
                    _ => Ok(reply("capable answer")),
                }
            },
        )
        .await;
        assert_eq!(
            outcome.is_ok_and(|(selected, _)| selected == "capable"),
            escalates
        );
    }
    Ok(())
}

fn recovering_classifier() -> super::VgrClassifier {
    let mut config = VgrConfig::new(ModelId::new("efficient"), ModelId::new("capable"));
    config.mode = ServingMode::Evaluate;
    config.breaker.threshold = 1;
    config.breaker.cooldown = std::time::Duration::ZERO;
    let classifier = super::VgrClassifier {
        breaker: super::CircuitBreaker::new(config.breaker),
        config,
    };
    classifier.breaker.failure();
    classifier
}

#[tokio::test]
async fn verifier_budget_exhaustion_is_not_an_endpoint_failure() -> Result<()> {
    use std::time::{Duration, Instant};

    use crate::core::algorithm::{Driver, RuntimeModels, Step};

    let mut config = VgrConfig::new(ModelId::new("efficient"), ModelId::new("capable"));
    config.deadline = Duration::from_millis(100);
    config.breaker.threshold = 1;
    config.breaker.cooldown = Duration::from_secs(60);
    assert!(config.targets.judge.is_none());
    let classifier = super::VgrClassifier {
        breaker: super::CircuitBreaker::new(config.breaker),
        config,
    };
    let mut state = State::default();
    let task = session(vec![Message::text(Role::User, "list files")]);
    let LlmResponse::Agg(proposed) = complete_call().llm_response else {
        unreachable!();
    };
    let (driver, mut steps) = Driver::new("vgr", Arc::new(RuntimeModels::default()));
    let context = super::RequestContext {
        driver: &driver,
        request: &task,
        started: Instant::now(),
    };
    let mut judging = Box::pin(classifier.verify_turn(&context, &proposed, &mut state));
    assert!(futures::poll!(&mut judging).is_pending());
    let Step::CallModel(pending_call) = steps.try_recv().expect("judge call should be queued")?
    else {
        unreachable!();
    };
    assert_eq!(pending_call.models, vec![ModelId::new("efficient")]);
    let escalates = judging.await;
    drop(pending_call);
    assert!(!escalates);
    assert!(super::turn_streak(&state) < super::TURN_CONFIRMATIONS);
    assert!(classifier.breaker.attempt().is_some());
    Ok(())
}

#[tokio::test]
async fn skipped_requests_leave_the_recovery_trial_available() -> Result<()> {
    use crate::core::classifier::Classifier;
    use crate::core::testing::empty_driver;
    use switchyard_protocol::ImageSource;

    for skip in ["image", "latched", "off", "kill", "deadline"] {
        let mut classifier = recovering_classifier();
        let mut state = State::default();
        let mut task = request(vec![Message::text(Role::User, "task")]);
        match skip {
            "image" => {
                task.llm_request.messages[0].content = vec![ContentBlock::Image {
                    source: ImageSource::Url {
                        url: "https://example.test/image.png".into(),
                        detail: None,
                    },
                }]
            }
            "latched" => {
                state.extra.insert(
                    super::TURN_STREAK_KEY.to_string(),
                    StateValue::Count(super::TURN_CONFIRMATIONS),
                );
            }
            "off" => classifier.config.mode = ServingMode::Off,
            "kill" => {
                let switch = super::super::KillSwitch::new();
                switch.engage();
                classifier.config.kill_switch = Some(switch);
            }
            "deadline" => classifier.config.deadline = std::time::Duration::ZERO,
            _ => unreachable!(),
        }
        classifier
            .score(&mut state, &mut task, &empty_driver())
            .await?;
        assert!(
            classifier.breaker.attempt().is_some(),
            "{skip} reserved the recovery trial"
        );
    }
    Ok(())
}

#[tokio::test]
async fn cancelling_efficient_generation_releases_the_recovery_trial() -> Result<()> {
    use crate::core::algorithm::{Driver, RuntimeModels};
    use crate::core::classifier::Classifier;

    let classifier = recovering_classifier();
    let mut state = State::default();
    let mut task = request(vec![Message::text(Role::User, "task")]);
    let (driver, _steps) = Driver::new("vgr", Arc::new(RuntimeModels::default()));
    let mut pending = classifier.score(&mut state, &mut task, &driver);
    assert!(futures::poll!(&mut pending).is_pending());
    assert!(classifier.breaker.attempt().is_none());
    drop(pending);
    assert!(classifier.breaker.attempt().is_some());
    Ok(())
}

#[tokio::test]
async fn decisions_attach_the_reason_and_exclude_efficient_capable_fallbacks() -> Result<()> {
    use crate::core::algorithm::Call;
    use crate::core::testing::serve_decision;

    for (mode, reason, served) in [
        (ServingMode::Off, "mode_off", "capable"),
        (ServingMode::Evaluate, "decided", "efficient"),
        (ServingMode::Shadow, "decided", "capable"),
    ] {
        let algorithm = route(|config| config.mode = mode)?;
        let outcome = crate::drive(
            algorithm,
            request(vec![Message::text(Role::User, "task")]),
            Arc::new(category_models(Category::Any, &["efficient", "capable"]).into()),
            |call| async move {
                match call {
                    Call::Model(call) => {
                        let response = if call.models[0] == ModelId::new("efficient") {
                            reply("done")
                        } else {
                            scored(0.9)
                        };
                        call.respond(std::future::ready(Ok(response))).await
                    }
                    Call::Decision(call) => serve_decision(*call).await,
                }
            },
        )
        .await?;
        assert_eq!(outcome.selected_model_ids, vec![ModelId::new(served)]);
        let evidence = outcome
            .metadata
            .and_then(|metadata| metadata.evidence)
            .unwrap_or_default();
        assert_eq!(evidence["source"], "vgr");
        assert_eq!(evidence["reason_code"], reason);
        assert_eq!(evidence["verdict"], served);
        assert_eq!(evidence["policy_version"], "1.3");
        if reason == "decided" {
            assert_eq!(evidence["branch"], "DefaultVerified");
        }
    }
    Ok(())
}

#[tokio::test]
async fn capable_only_work_does_not_get_an_efficient_handoff_notice() -> Result<()> {
    for scenario in ["shadow", "kill_switch", "efficient_turn_budget"] {
        let route = route(|config| {
            config.agentic_handoff = true;
            match scenario {
                "shadow" => config.mode = ServingMode::Shadow,
                "kill_switch" => {
                    let switch = super::super::safety::KillSwitch::new();
                    switch.engage();
                    config.kill_switch = Some(switch);
                }
                _ => config.efficient_turn_budget = Some(std::time::Duration::ZERO),
            }
        })?;
        let task = session(vec![
            Message::text(Role::User, "fix the build"),
            call("c1"),
            result("c1", "ok"),
        ]);
        let (selected, _) = test_drive_with_models(
            route,
            task,
            category_models(Category::Any, &["efficient", "capable"]),
            move |target: ModelId, request: Request| async move {
                Ok(match target.as_str() {
                    "efficient" => complete_call(),
                    "judge" => scored(0.0),
                    _ => {
                        let text = serde_json::to_string(&request.llm_request.messages)
                            .unwrap_or_default();
                        assert!(!text.contains("Routing notice"), "{scenario}");
                        reply("capable answer")
                    }
                })
            },
        )
        .await?;
        assert_eq!(selected, "capable");
    }
    Ok(())
}

#[tokio::test]
async fn latched_sessions_do_not_repeat_the_handoff_on_later_user_turns() -> Result<()> {
    let route = route(|config| config.agentic_handoff = true)?;
    assert_eq!(
        drive(
            &route,
            session(vec![Message::text(Role::User, "fix the build")]),
            Arc::new(Calls::default()),
            complete_call,
            0.9,
        )
        .await?,
        "efficient"
    );
    let sent = Arc::new(parking_lot::Mutex::new(Vec::new()));
    for new_user_turn in [false, true] {
        let mut messages = vec![
            Message::text(Role::User, "fix the build"),
            call("c1"),
            result("c1", "ok"),
        ];
        if new_user_turn {
            messages.push(Message::text(Role::User, "now run the tests"));
        }
        let sent = Arc::clone(&sent);
        let (selected, _) = test_drive_with_models(
            Arc::clone(&route),
            session(messages),
            category_models(Category::Any, &["efficient", "capable"]),
            move |target: ModelId, request: Request| {
                let sent = Arc::clone(&sent);
                async move {
                    Ok(match target.as_str() {
                        "efficient" => complete_call(),
                        "judge" => scored(0.9),
                        _ => {
                            sent.lock().push(
                                serde_json::to_string(&request.llm_request.messages)
                                    .unwrap_or_default(),
                            );
                            reply("capable answer")
                        }
                    })
                }
            },
        )
        .await?;
        assert_eq!(selected, "capable");
    }
    let sent = sent.lock();
    assert_eq!(sent.len(), 2);
    assert!(sent[0].contains("Routing notice"));
    assert!(!sent[1].contains("Routing notice"));
    Ok(())
}

#[tokio::test]
async fn compact_handoff_is_optional_and_replays_before_retained_capable_routing() -> Result<()> {
    let previous = vec![
        Message::text(Role::User, "inspect the repository"),
        call("old"),
        result("old", "previous output"),
        Message::text(Role::Assistant, "inspection complete"),
    ];
    for compact_handoff in [None, Some(false), Some(true)] {
        let route = route(|config| {
            config.agentic_handoff = true;
            if let Some(enabled) = compact_handoff {
                config.compact_handoff = enabled;
            }
        })?;
        assert_eq!(
            drive(
                &route,
                session(
                    previous
                        .iter()
                        .cloned()
                        .chain([Message::text(Role::User, "fix the build")])
                        .collect(),
                ),
                Arc::new(Calls::default()),
                complete_call,
                0.0,
            )
            .await?,
            "efficient"
        );
        let sent = Arc::new(parking_lot::Mutex::new(Vec::new()));
        let efficient_calls = Arc::new(AtomicUsize::new(0));
        for follow_up in [false, true] {
            let mut messages = previous.clone();
            messages.extend([
                Message::text(Role::User, "fix the build"),
                call("c1"),
                result("c1", "ok"),
            ]);
            if follow_up {
                messages.extend([call("capable-1"), result("capable-1", "capable output")]);
            }
            let sent = Arc::clone(&sent);
            let efficient_calls = Arc::clone(&efficient_calls);
            let (selected, _) = test_drive_with_models(
                Arc::clone(&route),
                session(messages),
                category_models(Category::Any, &["efficient", "capable"]),
                move |target: ModelId, request: Request| {
                    let sent = Arc::clone(&sent);
                    let efficient_calls = Arc::clone(&efficient_calls);
                    async move {
                        Ok(match target.as_str() {
                            "efficient" => {
                                efficient_calls.fetch_add(1, Ordering::Relaxed);
                                reply("the build passes")
                            }
                            "judge" => scored(0.0),
                            "capable" => {
                                sent.lock().push(request.llm_request.messages);
                                reply("capable answer")
                            }
                            _ => reply("no"),
                        })
                    }
                },
            )
            .await?;
            assert_eq!(selected, "capable");
        }
        assert_eq!(efficient_calls.load(Ordering::Relaxed), 1);
        let sent = sent.lock();
        assert_eq!(sent.len(), 2);
        if compact_handoff == Some(true) {
            assert_eq!(sent[0].len(), 5);
            assert_eq!(&sent[0][..4], previous.as_slice());
            assert_eq!(sent[1].len(), 7);
            assert_eq!(sent[0], sent[1][..5]);
            let digest = sent[0][4].text_content("").unwrap_or_default();
            assert!(digest.starts_with("fix the build"));
            assert!(digest.contains("<condensed_history>"));
            assert!(digest.contains("the build passes"));
            assert!(digest.contains("Routing notice"));
            assert_eq!(sent[1][5], call("capable-1"));
            assert_eq!(sent[1][6], result("capable-1", "capable output"));
        } else {
            let mut original = previous.clone();
            original.extend([
                Message::text(Role::User, "fix the build"),
                call("c1"),
                result("c1", "ok"),
            ]);
            assert_eq!(&sent[0][..7], original.as_slice());
            assert_eq!(sent[0].len(), 8);
            assert!(
                sent[0][7]
                    .text_content("")
                    .unwrap_or_default()
                    .contains("Routing notice")
            );
            let mut follow_up = original;
            follow_up.extend([call("capable-1"), result("capable-1", "capable output")]);
            assert_eq!(sent[1], follow_up);
        }
    }
    Ok(())
}

#[tokio::test]
async fn an_escalated_agentic_run_hands_off_once_with_the_unverified_claim() -> Result<()> {
    let route = route(|config| config.agentic_handoff = true)?;
    assert_eq!(
        drive(
            &route,
            session(vec![Message::text(Role::User, "fix the build")]),
            Arc::new(Calls::default()),
            complete_call,
            0.0,
        )
        .await?,
        "efficient"
    );
    let sent = Arc::new(parking_lot::Mutex::new(Vec::new()));
    let task = || {
        session(vec![
            Message::text(Role::User, "fix the build"),
            call("c1"),
            result("c1", "ok"),
        ])
    };
    for _ in 0..2 {
        let sent = Arc::clone(&sent);
        test_drive_with_models(
            Arc::clone(&route),
            task(),
            category_models(Category::Any, &["efficient", "capable"]),
            move |target: ModelId, request: Request| {
                let sent = Arc::clone(&sent);
                async move {
                    Ok(match target.as_str() {
                        "efficient" => reply("the build passes"),
                        "judge" => scored(0.0),
                        "capable" => {
                            let text = serde_json::to_string(&request.llm_request.messages)
                                .unwrap_or_default();
                            sent.lock().push(text);
                            reply("capable answer")
                        }
                        _ => reply("no"),
                    })
                }
            },
        )
        .await?;
    }
    let sent = sent.lock();
    assert_eq!(sent.len(), 2);
    assert!(sent[0].contains("Routing notice") && sent[0].contains("the build passes"));
    assert!(
        !sent[1].contains("Routing notice"),
        "sent once per user turn"
    );
    Ok(())
}

#[tokio::test]
async fn a_spent_efficient_turn_budget_skips_the_efficient_attempt() -> Result<()> {
    let route = route(|config| config.efficient_turn_budget = Some(std::time::Duration::ZERO))?;
    let calls = Arc::new(Calls::default());
    let task = session(vec![Message::text(Role::User, "fix the build")]);
    assert_eq!(
        drive(&route, task, calls.clone(), complete_call, 0.0).await?,
        "capable"
    );
    assert_eq!(calls.0.load(Ordering::Relaxed), 0);
    Ok(())
}

#[test]
fn handoff_preserves_existing_messages_and_non_text_content() {
    use switchyard_protocol::ImageSource;

    let mut classifier = recovering_classifier();
    classifier.config.agentic_handoff = true;
    let mut task = request(vec![
        Message::text(Role::User, "finish the task"),
        Message::text(Role::User, "keep the original files"),
        call("c1"),
        result("c1", "the command failed"),
        Message::text(Role::User, "preserve this requirement"),
        Message::text(Role::User, "continue"),
        Message::text(Role::User, "then report the result"),
    ]);
    if let ContentBlock::ToolResult(result) = &mut task.llm_request.messages[3].content[0] {
        result.content.push(ContentBlock::Image {
            source: ImageSource::Url {
                url: "https://example.test/result.png".into(),
                detail: None,
            },
        });
    }
    let original = task.llm_request.messages.clone();
    let mut state = State::default();
    super::set_flag(&mut state, super::EFFICIENT_SERVED_KEY, true);
    classifier.hand_off(&mut task, &mut state, Some("unchecked result"));
    // A notice may be appended, but all original content stays unchanged.
    let messages = &task.llm_request.messages;
    assert_eq!(messages.len(), original.len() + 1);
    assert_eq!(&messages[..original.len()], original.as_slice());
    let last = messages.last().expect("handoff note");
    assert!(
        last.text_content("")
            .is_some_and(|text| text.contains("unchecked result"))
    );
    let after_first = task.llm_request.messages.clone();
    classifier.hand_off(&mut task, &mut state, Some("duplicate notice"));
    assert_eq!(task.llm_request.messages, after_first);
}

#[tokio::test]
async fn a_blank_prompt_escalates_without_an_efficient_attempt() -> Result<()> {
    let continuing = route(|_| {})?;
    let route = route(|_| {})?;
    let calls = Arc::new(Calls::default());
    let blank_follow_up = vec![
        Message::text(Role::User, "what is 2 + 2?"),
        Message::text(Role::Assistant, "4"),
        Message::text(Role::User, " "),
    ];
    for messages in [
        vec![Message::text(Role::User, "  \n ")],
        vec![Message::text(Role::User, "\u{200B}\u{FEFF}")],
        blank_follow_up,
    ] {
        assert_eq!(
            drive(&route, session(messages), calls.clone(), complete_call, 0.0).await?,
            "capable"
        );
    }
    assert_eq!(calls.0.load(Ordering::Relaxed), 0);

    // Tool results sent as a user message continue the task; they are not a blank prompt.
    let mut results = result("a", "file.txt");
    results.role = Role::User;
    let continuation = session(vec![
        Message::text(Role::User, "list files"),
        call("a"),
        results,
    ]);
    drive(&continuing, continuation, calls.clone(), complete_call, 0.0).await?;
    assert_eq!(calls.0.load(Ordering::Relaxed), 1);
    Ok(())
}

#[tokio::test]
async fn an_agent_marked_nudge_keeps_a_single_escalation_on_capable() -> Result<()> {
    let route = route(|_| {})?;
    let calls = Arc::new(Calls::default());
    let truncated = || tool_call(json!(r#"{"command":"l"#), StopReason::MaxTokens);
    let task = session(vec![Message::text(Role::User, "fix the build")]);
    drive(&route, task, calls.clone(), truncated, 0.0).await?;
    let nudge = vec![
        Message::text(Role::User, "fix the build"),
        Message::text(Role::Assistant, "fixed"),
        Message::text(Role::User, "Verify the change before you stop."),
    ];
    let nudge = initiated_by(session(nudge), "agent");
    assert_eq!(
        drive(&route, nudge, calls.clone(), complete_call, 0.0).await?,
        "capable"
    );
    assert_eq!(calls.0.load(Ordering::Relaxed), 1);
    Ok(())
}

fn initiated_by(mut request: Request, initiator: &str) -> Request {
    let metadata = request.metadata.get_or_insert_with(Metadata::default);
    metadata
        .http_headers
        .get_or_insert_with(http::HeaderMap::new)
        .insert("x-initiator", initiator.parse().unwrap());
    request
}

#[tokio::test]
async fn a_held_follow_up_records_the_latch_as_its_reason() -> Result<()> {
    use crate::core::algorithm::Call;
    use crate::core::testing::serve_decision;

    let algorithm = route(|config| config.mode = ServingMode::Shadow)?;
    let models = Arc::new(category_models(Category::Any, &["efficient", "capable"]).into());
    let serve = |call| async move {
        match call {
            Call::Model(call) => call.respond(std::future::ready(Ok(reply("done")))).await,
            Call::Decision(call) => serve_decision(*call).await,
        }
    };
    let escalated = crate::drive(
        Arc::clone(&algorithm),
        session(vec![Message::text(Role::User, "fix the build")]),
        Arc::clone(&models),
        serve,
    )
    .await?;
    assert_eq!(escalated.selected_model_ids, vec![ModelId::new("capable")]);

    let held = crate::drive(
        algorithm,
        session(vec![
            Message::text(Role::User, "fix the build"),
            call("c1"),
            result("c1", "ok"),
        ]),
        models,
        serve,
    )
    .await?;
    assert_eq!(held.selected_model_ids, vec![ModelId::new("capable")]);
    let evidence = held
        .metadata
        .and_then(|metadata| metadata.evidence)
        .unwrap_or_default();
    assert_eq!(evidence["source"], "vgr");
    assert_eq!(evidence["reason_code"], "turn_verification_latched");
    assert_eq!(evidence["verdict"], "capable");
    Ok(())
}

#[tokio::test]
async fn shadow_rechecks_new_user_turns_with_session_latching_enabled() -> Result<()> {
    let route = route(|config| {
        config.mode = ServingMode::Shadow;
        config.latch_escalation = true;
    })?;
    let calls = Arc::new(Calls::default());
    for messages in [
        vec![Message::text(Role::User, "fix the build")],
        vec![
            Message::text(Role::User, "fix the build"),
            Message::text(Role::Assistant, "fixed"),
            Message::text(Role::User, "now add a test"),
        ],
    ] {
        assert_eq!(
            drive(&route, session(messages), calls.clone(), complete_call, 0.0).await?,
            "capable"
        );
    }
    assert_eq!(calls.0.load(Ordering::Relaxed), 2);
    Ok(())
}

#[tokio::test]
async fn an_escalation_latch_holds_the_session_past_new_questions() -> Result<()> {
    let route = route(|config| config.latch_escalation = true)?;
    let calls = Arc::new(Calls::default());
    let truncated = || tool_call(json!(r#"{"command":"l"#), StopReason::MaxTokens);
    let task = session(vec![Message::text(Role::User, "fix the build")]);
    assert_eq!(
        drive(&route, task, calls.clone(), truncated, 0.0).await?,
        "capable"
    );
    let follow_up = session(vec![
        Message::text(Role::User, "fix the build"),
        Message::text(Role::Assistant, "fixed"),
        Message::text(Role::User, "now add a test"),
    ]);
    assert_eq!(
        drive(&route, follow_up, calls.clone(), complete_call, 0.0).await?,
        "capable"
    );
    assert_eq!(calls.0.load(Ordering::Relaxed), 1);
    Ok(())
}

#[tokio::test]
async fn an_initiator_header_decides_when_the_turn_latch_releases() -> Result<()> {
    let route = route(|_| {})?;
    let calls = Arc::new(Calls::default());
    let task = || session(vec![Message::text(Role::User, "fix the build")]);
    drive(&route, task(), calls.clone(), complete_call, 0.9).await?;
    drive(&route, task(), calls.clone(), complete_call, 0.9).await?;
    assert_eq!(calls.0.load(Ordering::Relaxed), 2);

    // A client nudge after a final reply looks like a new question; marked agent, it keeps the latch.
    let nudge = vec![
        Message::text(Role::User, "fix the build"),
        Message::text(Role::Assistant, "fixed"),
        Message::text(Role::User, "Verify the change before you stop."),
    ];
    assert_eq!(
        drive(
            &route,
            initiated_by(session(nudge), "agent"),
            calls.clone(),
            complete_call,
            0.0
        )
        .await?,
        "capable"
    );
    assert_eq!(calls.0.load(Ordering::Relaxed), 2);

    // A person's correction after tool results looks like a continuation; marked user, it is verified afresh.
    let steer = vec![
        Message::text(Role::User, "fix the build"),
        call("c1"),
        result("c1", "ok"),
        Message::text(Role::User, "use the staging config instead"),
    ];
    assert_eq!(
        drive(
            &route,
            initiated_by(session(steer), "user"),
            calls.clone(),
            complete_call,
            0.0
        )
        .await?,
        "efficient"
    );
    assert_eq!(calls.0.load(Ordering::Relaxed), 3);
    Ok(())
}

#[tokio::test]
async fn tool_work_from_an_earlier_user_turn_is_not_handed_off() -> Result<()> {
    let route = route(|config| config.agentic_handoff = true)?;
    let (selected, _) = test_drive_with_models(
        route,
        session(vec![
            Message::text(Role::User, "fix the build"),
            call("c1"),
            result("c1", "ok"),
            Message::text(Role::Assistant, "the build passes"),
            Message::text(Role::User, "What is 41 plus 42?"),
        ]),
        category_models(Category::Any, &["efficient", "capable"]),
        |target: ModelId, request: Request| async move {
            Ok(match target.as_str() {
                "efficient" => reply("83"),
                "capable" => {
                    let text =
                        serde_json::to_string(&request.llm_request.messages).unwrap_or_default();
                    assert!(text.contains("41 plus 42"));
                    assert!(!text.contains("Routing notice"));
                    reply("83")
                }
                _ => scored(0.0),
            })
        },
    )
    .await?;
    assert_eq!(selected, "capable");
    Ok(())
}

#[tokio::test]
async fn a_capable_decision_falls_back_to_efficient_without_the_handoff_note() -> Result<()> {
    use crate::core::algorithm::Call;
    use crate::core::testing::serve_decision;

    let task = || {
        session(vec![
            Message::text(Role::User, "fix the build"),
            call("c1"),
            result("c1", "ok"),
        ])
    };
    let noted = |request: &Request| {
        serde_json::to_string(&request.llm_request.messages)
            .is_ok_and(|text| text.contains("Routing notice"))
    };
    for killed in [false, true] {
        let switch = super::super::KillSwitch::new();
        let algorithm = route(|config| {
            config.agentic_handoff = true;
            config.kill_switch = Some(switch.clone());
        })?;
        drive(
            &algorithm,
            session(vec![Message::text(Role::User, "fix the build")]),
            Arc::new(Calls::default()),
            complete_call,
            0.0,
        )
        .await?;
        if killed {
            switch.engage();
        }
        let outcome = crate::drive(
            algorithm,
            task(),
            Arc::new(category_models(Category::Any, &["efficient", "capable"]).into()),
            |call| async move {
                match call {
                    Call::Model(call) => {
                        let response = match call.models[0].as_str() {
                            "efficient" => reply("the build passes"),
                            "judge" => scored(0.0),
                            _ => reply("no"),
                        };
                        call.respond(std::future::ready(Ok(response))).await
                    }
                    Call::Decision(call) => serve_decision(*call).await,
                }
            },
        )
        .await?;
        assert!(noted(&outcome.request));
        if killed {
            assert_eq!(outcome.selected_model_ids, vec![ModelId::new("capable")]);
            assert!(outcome.fallback_request.is_none());
        } else {
            assert_eq!(
                outcome.selected_model_ids,
                vec![ModelId::new("capable"), ModelId::new("efficient")]
            );
            let fallback = outcome
                .fallback_request
                .expect("an efficient fallback request");
            assert_eq!(fallback.llm_request.messages, task().llm_request.messages);
        }
    }
    Ok(())
}

#[tokio::test]
async fn default_verification_asks_capable_evidence_then_answer() -> Result<()> {
    for (evidence, answer, expected) in [
        ("yes", "yes", "efficient"),
        ("yes", "no", "capable"),
        ("yes", "unclear", "capable"),
        ("no", "yes", "capable"),
        ("unclear", "yes", "capable"),
    ] {
        let route = route(|_| {})?;
        let calls = Arc::new(parking_lot::Mutex::new(Vec::new()));
        let seen = Arc::clone(&calls);
        let (selected, _) = test_drive_with_models(
            route,
            session(vec![Message::text(Role::User, "explain quicksort")]),
            category_models(Category::Any, &["efficient", "capable"]),
            move |target: ModelId, request: Request| {
                let seen = Arc::clone(&seen);
                async move {
                    Ok(match target.as_str() {
                        "efficient" => reply("Choose a pivot and partition the input."),
                        "judge" => scored(0.1),
                        "capable-judge" => {
                            let is_answer = request
                                .llm_request
                                .instructions
                                .iter()
                                .flat_map(|block| &block.content)
                                .any(|block| {
                                    matches!(block, ContentBlock::Text { text }
                                    if text.starts_with("You are a strict answer verifier"))
                                });
                            seen.lock().push(is_answer);
                            reply(if is_answer { answer } else { evidence })
                        }
                        _ => reply("capable answer"),
                    })
                }
            },
        )
        .await?;
        assert_eq!(selected.as_str(), expected, "{evidence}, {answer}");
        assert_eq!(
            *calls.lock(),
            if evidence == "yes" {
                vec![false, true]
            } else {
                vec![false]
            }
        );
    }
    Ok(())
}

#[tokio::test]
async fn shadow_keeps_judging_a_turn_until_vgr_escalates_it() -> Result<()> {
    fn truncated() -> Response {
        tool_call(json!(r#"{"command":"l"#), StopReason::MaxTokens)
    }
    for latch_escalation in [false, true] {
        let route = route(|config| {
            config.mode = ServingMode::Shadow;
            config.latch_escalation = latch_escalation;
        })?;
        let calls = Arc::new(Calls::default());
        let mut messages = vec![Message::text(Role::User, "fix the build")];
        for (step, efficient) in [complete_call, complete_call, truncated, complete_call]
            .into_iter()
            .enumerate()
        {
            assert_eq!(
                drive(
                    &route,
                    session(messages.clone()),
                    calls.clone(),
                    efficient,
                    0.1
                )
                .await?,
                "capable"
            );
            let id = format!("c{step}");
            messages.extend([call(&id), result(&id, "ok")]);
        }
        assert_eq!(calls.0.load(Ordering::Relaxed), 3);
        messages.push(Message::text(Role::User, "keep going"));
        drive(
            &route,
            initiated_by(session(messages.clone()), "agent"),
            calls.clone(),
            complete_call,
            0.1,
        )
        .await?;
        assert_eq!(calls.0.load(Ordering::Relaxed), 3);
        drive(
            &route,
            initiated_by(session(messages), "user"),
            calls.clone(),
            complete_call,
            0.1,
        )
        .await?;
        assert_eq!(calls.0.load(Ordering::Relaxed), 4);
    }
    Ok(())
}

#[tokio::test]
async fn shadow_final_answer_commits_do_not_latch_but_rejections_do() -> Result<()> {
    let route = route(|config| {
        config.mode = ServingMode::Shadow;
        config.targets.capable_judge = None;
    })?;
    let calls = Arc::new(Calls::default());
    let mut messages = vec![Message::text(Role::User, "explain quicksort")];
    for probability in [0.9, 0.9, 0.1, 0.9] {
        let task = initiated_by(session(messages.clone()), "agent");
        assert_eq!(
            drive(
                &route,
                task,
                calls.clone(),
                || reply("Partition around a pivot."),
                probability
            )
            .await?,
            "capable"
        );
        messages.push(Message::text(Role::Assistant, "Partition around a pivot."));
        messages.push(Message::text(Role::User, "continue"));
    }
    assert_eq!(calls.0.load(Ordering::Relaxed), 3);
    Ok(())
}

#[tokio::test]
async fn client_recovery_messages_keep_the_turn_latch() -> Result<()> {
    let route = route(|_| {})?;
    assert!(route.tracks_answers());
    let calls = Arc::new(Calls::default());
    let task = || session(vec![Message::text(Role::User, "fix the build")]);
    for _ in 0..2 {
        drive(&route, task(), calls.clone(), complete_call, 0.9).await?;
    }
    // Hermes-style recovery: a nudge after an empty answer, and one sent the moment a
    // complete-looking answer promised a tool call it never made.
    for (final_reply, previous, nudge) in [
        (
            false,
            "(empty)",
            "You just executed tool calls but returned an empty response.",
        ),
        (
            true,
            "Let me run the tests.",
            "Your previous turn indicated a tool call but none was included.",
        ),
    ] {
        route
            .observe_answer("session-1", AnswerSummary { final_reply })
            .await;
        let recovery = session(vec![
            Message::text(Role::User, "fix the build"),
            Message::text(Role::Assistant, previous),
            Message::text(Role::User, nudge),
        ]);
        assert_eq!(
            drive(&route, recovery, calls.clone(), complete_call, 0.0).await?,
            "capable"
        );
    }
    assert_eq!(calls.0.load(Ordering::Relaxed), 2);
    let next = initiated_by(
        session(vec![
            Message::text(Role::User, "fix the build"),
            Message::text(Role::Assistant, "Let me run the tests."),
            Message::text(Role::User, "now do another task"),
        ]),
        "user",
    );
    assert_eq!(
        drive(&route, next, calls.clone(), complete_call, 0.0).await?,
        "efficient"
    );
    assert_eq!(calls.0.load(Ordering::Relaxed), 3);
    Ok(())
}

#[test]
fn only_a_paused_reply_to_a_reported_final_answer_releases_the_latch() {
    let messages = [
        Message::text(Role::User, "fix the build"),
        Message::text(Role::Assistant, "fixed"),
        Message::text(Role::User, "now add a test"),
    ];
    let now = unix_seconds();
    for (reported, releases) in [
        (None, true),
        (Some(0), false),
        (Some(now), false),
        (Some(now - FOLLOW_UP_QUIET_SECS), true),
    ] {
        let mut state = State::default();
        if let Some(finished) = reported {
            state
                .extra
                .insert(LAST_ANSWER_FINAL_KEY.into(), StateValue::Count(finished));
        }
        assert_eq!(
            answers_final_reply(&state, &messages),
            releases,
            "{reported:?}"
        );
    }
}

#[tokio::test]
async fn a_failed_capable_verifier_still_offers_the_efficient_answer() -> Result<()> {
    use crate::core::algorithm::Call;
    use crate::core::testing::serve_decision;

    let algorithm = route(|config| config.task_typing = true)?;
    let task = || session(vec![Message::text(Role::User, "What is 1+1?")]);
    // Exercise production error propagation through CallModel::respond.
    let outcome = crate::drive(
        algorithm,
        task(),
        Arc::new(category_models(Category::Any, &["efficient", "capable"]).into()),
        |call| async move {
            match call {
                Call::Model(call) => {
                    let target = call.models[0].clone();
                    let readout = call
                        .request
                        .llm_request
                        .extensions
                        .fields
                        .contains_key("logprobs");
                    let response = match target.as_str() {
                        "efficient" => Ok(reply("2")),
                        "judge" if readout => Ok(scored(0.05)),
                        "judge" => Ok(reply("Answer")),
                        _ => Err(crate::LibsyError::client_call(
                            target,
                            LlmClientError::UpstreamHttp {
                                status: http::StatusCode::BAD_GATEWAY,
                                body: String::new(),
                            },
                        )),
                    };
                    call.respond(std::future::ready(response)).await
                }
                Call::Decision(call) => serve_decision(*call).await,
            }
        },
    )
    .await?;
    assert_eq!(
        outcome.selected_model_ids,
        vec![ModelId::new("capable"), ModelId::new("efficient")]
    );
    let fallback = outcome
        .fallback_request
        .expect("an efficient fallback request");
    assert_eq!(fallback.llm_request.messages, task().llm_request.messages);
    Ok(())
}

#[tokio::test]
async fn a_step_that_cannot_commit_over_a_tool_error_skips_local_checks() -> Result<()> {
    let failed = session(vec![
        Message::text(Role::User, "What is happening today?"),
        call("c1"),
        result("c1", r#"{"error": "search-only backend"}"#),
    ]);
    // A typed answer with tool use in its turn is judged as chat, which needs a clean record.
    for (typed, expected) in [
        ("Answer", vec!["judge", "efficient", "capable"]),
        ("Chat", vec!["judge", "capable"]),
    ] {
        let calls = Arc::new(parking_lot::Mutex::new(Vec::new()));
        let seen = Arc::clone(&calls);
        let (selected, _) = test_drive_with_models(
            route(|config| config.task_typing = true)?,
            failed.clone(),
            category_models(Category::Any, &["efficient", "capable"]),
            move |target: ModelId, _| {
                seen.lock().push(target.to_string());
                std::future::ready(Ok(reply(typed)))
            },
        )
        .await?;
        assert_eq!(selected.as_str(), "capable");
        assert_eq!(*calls.lock(), expected, "typed {typed}");
    }
    Ok(())
}