Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust
36.4 KB
Raw Blame
//! Ordered LLM provider fallback chain.
//!
//! See `docs/llm-provider-fallback.md` (issue #648) for the full design.
//! `FallbackLlmProvider` wraps an ordered list of already-constructed
//! providers behind the same [`LlmProvider`] trait: on a transient failure
//! (`LlmError::is_transient()`) it advances to the next eligible candidate,
//! preserving the original request, schema, and operation id; a
//! deterministic failure (bad request, auth, unsupported schema, malformed
//! response) returns immediately without trying the rest of the chain.
//! `Config::llm_provider_chain` in `ai-memory-cli` is the sole place that
//! builds one; `build_provider` remains the sole construction path for each
//! candidate.

use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};

use async_trait::async_trait;
use jiff::Timestamp;
use tracing::{debug, info, warn};

use crate::error::{LlmError, LlmResult};
use crate::health::CandidateHealth;
use crate::provider::LlmProvider;
use crate::types::{ChatRequest, ChatResponse, LlmOperationId};

/// How long a candidate's circuit stays open after a transient failure
/// before it is eligible again. Restarting the process clears all circuit
/// state; there is no durable or cross-process cooldown.
pub const CIRCUIT_COOLDOWN: Duration = Duration::from_secs(30);
const MAX_CIRCUIT_COOLDOWN: Duration = Duration::from_secs(60 * 60);

/// One provider in an ordered fallback chain.
///
/// Carries only the diagnostic labels used by health reporting and circuit
/// tracking — never the raw configuration or credentials that built
/// `inner`. `build_provider` remains the sole construction path; this type
/// only wraps its result.
pub struct Candidate {
    provider: &'static str,
    model: String,
    inner: Arc<dyn LlmProvider>,
}

impl Candidate {
    /// Wrap an already-constructed provider for use in a fallback chain.
    #[must_use]
    pub fn new(
        provider: &'static str,
        model: impl Into<String>,
        inner: Arc<dyn LlmProvider>,
    ) -> Self {
        Self {
            provider,
            model: model.into(),
            inner,
        }
    }
}

/// Per-candidate mutable state: circuit + last-call bookkeeping, shared
/// between the call path and health reporting.
#[derive(Default)]
struct CandidateState {
    last_success_at: Option<Timestamp>,
    last_error_at: Option<Timestamp>,
    last_error_status: Option<u16>,
    last_error_class: Option<&'static str>,
    circuit_open_until: Option<Instant>,
}

struct CandidateEntry {
    candidate: Candidate,
    state: Mutex<CandidateState>,
}

/// Ordered LLM provider fallback chain.
///
/// Tries each eligible candidate in declaration order, advancing only on a
/// transient failure; a deterministic failure returns immediately. See the
/// module docs and `docs/llm-provider-fallback.md`.
pub struct FallbackLlmProvider {
    entries: Vec<CandidateEntry>,
    /// Index of the candidate that answered the most recently completed
    /// call, for health reporting.
    last_selected: Mutex<Option<usize>>,
}

impl FallbackLlmProvider {
    /// Build a chain from an ordered list of candidates (primary first,
    /// then fallbacks in declaration order).
    #[must_use]
    pub fn new(candidates: Vec<Candidate>) -> Self {
        Self {
            entries: candidates
                .into_iter()
                .map(|candidate| CandidateEntry {
                    candidate,
                    state: Mutex::new(CandidateState::default()),
                })
                .collect(),
            last_selected: Mutex::new(None),
        }
    }

    fn circuit_open(&self, index: usize) -> bool {
        let state = self.entries[index]
            .state
            .lock()
            .unwrap_or_else(|e| e.into_inner());
        matches!(state.circuit_open_until, Some(until) if Instant::now() < until)
    }

    fn record_success(&self, index: usize) {
        {
            let mut state = self.entries[index]
                .state
                .lock()
                .unwrap_or_else(|e| e.into_inner());
            state.last_success_at = Some(Timestamp::now());
            state.circuit_open_until = None;
        }
        let previous = self
            .last_selected
            .lock()
            .unwrap_or_else(|e| e.into_inner())
            .replace(index);
        // Logged on a change of answering candidate only, so a primary that
        // stays down for an hour yields one line instead of one per call.
        let switched = match previous {
            Some(previous) => previous != index,
            None => index > 0,
        };
        if switched {
            let candidate = &self.entries[index].candidate;
            info!(
                provider = candidate.provider,
                model = %candidate.model,
                position = index,
                "llm fallback chain now answering from this candidate"
            );
        }
    }

    fn record_error(&self, index: usize, err: &LlmError) {
        let cooldown = err.is_transient().then(|| {
            err.retry_after()
                .unwrap_or(CIRCUIT_COOLDOWN)
                .max(CIRCUIT_COOLDOWN)
                .min(MAX_CIRCUIT_COOLDOWN)
        });
        let should_log_transient = {
            let mut state = self.entries[index]
                .state
                .lock()
                .unwrap_or_else(|e| e.into_inner());
            let circuit_was_open =
                matches!(state.circuit_open_until, Some(until) if Instant::now() < until);
            state.last_error_at = Some(Timestamp::now());
            state.last_error_status = err.http_status();
            state.last_error_class = Some(err.class());
            if let Some(cooldown) = cooldown {
                state.circuit_open_until = Some(Instant::now() + cooldown);
            }
            cooldown.is_some() && !circuit_was_open
        };
        // Redacted fields only, as the reranker does: the `Display` of a
        // provider failure carries the upstream response body. The model is
        // the failing candidate's, which the caller cannot see: every caller
        // labels the chain with its first candidate.
        let candidate = &self.entries[index].candidate;
        match cooldown {
            Some(cooldown) if should_log_transient => warn!(
                provider = candidate.provider,
                model = %candidate.model,
                position = index,
                error_class = err.class(),
                error_status = ?err.http_status(),
                cooldown_secs = cooldown.as_secs(),
                "llm fallback candidate failed; skipping it until its cooldown ends"
            ),
            Some(_) => {}
            None => debug!(
                provider = candidate.provider,
                model = %candidate.model,
                position = index,
                error_class = err.class(),
                error_status = ?err.http_status(),
                "llm fallback candidate failed deterministically; the chain stops here"
            ),
        }
    }

    fn describe_failure(&self, index: usize, err: &LlmError) -> String {
        let candidate = &self.entries[index].candidate;
        match err.http_status() {
            Some(status) => format!(
                "{}/{}: {} {status}",
                candidate.provider,
                candidate.model,
                err.class()
            ),
            None => format!(
                "{}/{}: {}",
                candidate.provider,
                candidate.model,
                err.class()
            ),
        }
    }

    fn exhausted(&self, attempted: usize, summary: Vec<String>) -> LlmError {
        LlmError::AllCandidatesFailed {
            attempted,
            summary: summary.join("; "),
        }
    }
}

#[async_trait]
impl LlmProvider for FallbackLlmProvider {
    fn name(&self) -> &'static str {
        self.entries
            .first()
            .map_or("fallback-chain-empty", |entry| entry.candidate.provider)
    }

    fn model(&self) -> &str {
        self.entries
            .first()
            .map_or("", |entry| entry.candidate.model.as_str())
    }

    async fn complete(&self, request: ChatRequest) -> LlmResult<ChatResponse> {
        let mut attempted = 0usize;
        let mut summary = Vec::new();
        for (i, entry) in self.entries.iter().enumerate() {
            if self.circuit_open(i) {
                continue;
            }
            attempted += 1;
            match entry.candidate.inner.complete(request.clone()).await {
                Ok(response) => {
                    self.record_success(i);
                    return Ok(response);
                }
                Err(err) => {
                    let transient = err.is_transient();
                    summary.push(self.describe_failure(i, &err));
                    self.record_error(i, &err);
                    if !transient {
                        return Err(err);
                    }
                }
            }
        }
        Err(self.exhausted(attempted, summary))
    }

    async fn complete_with_operation_id(
        &self,
        request: ChatRequest,
        operation_id: LlmOperationId,
    ) -> LlmResult<ChatResponse> {
        let mut attempted = 0usize;
        let mut summary = Vec::new();
        for (i, entry) in self.entries.iter().enumerate() {
            if self.circuit_open(i) {
                continue;
            }
            attempted += 1;
            match entry
                .candidate
                .inner
                .complete_with_operation_id(request.clone(), operation_id)
                .await
            {
                Ok(response) => {
                    self.record_success(i);
                    return Ok(response);
                }
                Err(err) => {
                    let transient = err.is_transient();
                    summary.push(self.describe_failure(i, &err));
                    self.record_error(i, &err);
                    if !transient {
                        return Err(err);
                    }
                }
            }
        }
        Err(self.exhausted(attempted, summary))
    }

    async fn complete_structured_raw(
        &self,
        request: ChatRequest,
        schema: serde_json::Value,
    ) -> LlmResult<serde_json::Value> {
        let mut attempted = 0usize;
        let mut summary = Vec::new();
        for (i, entry) in self.entries.iter().enumerate() {
            if self.circuit_open(i) {
                continue;
            }
            attempted += 1;
            match entry
                .candidate
                .inner
                .complete_structured_raw(request.clone(), schema.clone())
                .await
            {
                Ok(value) => {
                    self.record_success(i);
                    return Ok(value);
                }
                Err(err) => {
                    let transient = err.is_transient();
                    summary.push(self.describe_failure(i, &err));
                    self.record_error(i, &err);
                    if !transient {
                        return Err(err);
                    }
                }
            }
        }
        Err(self.exhausted(attempted, summary))
    }

    async fn complete_structured_raw_with_operation_id(
        &self,
        request: ChatRequest,
        schema: serde_json::Value,
        operation_id: LlmOperationId,
    ) -> LlmResult<serde_json::Value> {
        let mut attempted = 0usize;
        let mut summary = Vec::new();
        for (i, entry) in self.entries.iter().enumerate() {
            if self.circuit_open(i) {
                continue;
            }
            attempted += 1;
            match entry
                .candidate
                .inner
                .complete_structured_raw_with_operation_id(
                    request.clone(),
                    schema.clone(),
                    operation_id,
                )
                .await
            {
                Ok(value) => {
                    self.record_success(i);
                    return Ok(value);
                }
                Err(err) => {
                    let transient = err.is_transient();
                    summary.push(self.describe_failure(i, &err));
                    self.record_error(i, &err);
                    if !transient {
                        return Err(err);
                    }
                }
            }
        }
        Err(self.exhausted(attempted, summary))
    }

    fn candidate_health(&self) -> Vec<CandidateHealth> {
        let selected = *self.last_selected.lock().unwrap_or_else(|e| e.into_inner());
        self.entries
            .iter()
            .enumerate()
            .map(|(i, entry)| {
                let state = entry.state.lock().unwrap_or_else(|e| e.into_inner());
                let now = Instant::now();
                CandidateHealth {
                    provider: entry.candidate.provider.to_string(),
                    model: entry.candidate.model.clone(),
                    last_selected: selected == Some(i),
                    last_success_at: state.last_success_at,
                    last_error_at: state.last_error_at,
                    last_error_status: state.last_error_status,
                    last_error_class: state.last_error_class.map(str::to_string),
                    // Only report while the cooldown genuinely has not
                    // elapsed yet — a stale past instant (no later call
                    // resets it) would otherwise read as "still open".
                    circuit_open_until: state.circuit_open_until.and_then(|until| {
                        (until > now)
                            .then(|| Timestamp::now() + until.saturating_duration_since(now))
                    }),
                }
            })
            .collect()
    }
}

#[cfg(test)]
mod tests {
    use std::sync::atomic::{AtomicUsize, Ordering};

    use super::*;
    use crate::types::{ChatMessage, Usage};

    /// A scripted provider double: returns each configured outcome in
    /// order (repeating the last one once exhausted), and records every
    /// request/schema/operation id it was called with.
    struct ScriptedLlm {
        name: &'static str,
        model: String,
        outcomes: Mutex<Vec<LlmResult<ChatResponse>>>,
        calls: AtomicUsize,
        seen_requests: Mutex<Vec<ChatRequest>>,
        seen_schemas: Mutex<Vec<serde_json::Value>>,
        seen_operation_ids: Mutex<Vec<Option<LlmOperationId>>>,
    }

    impl ScriptedLlm {
        fn new(name: &'static str, model: &str, outcomes: Vec<LlmResult<ChatResponse>>) -> Self {
            Self {
                name,
                model: model.to_string(),
                outcomes: Mutex::new(outcomes),
                calls: AtomicUsize::new(0),
                seen_requests: Mutex::new(Vec::new()),
                seen_schemas: Mutex::new(Vec::new()),
                seen_operation_ids: Mutex::new(Vec::new()),
            }
        }

        fn ok(name: &'static str, model: &str, text: &str) -> Self {
            Self::new(
                name,
                model,
                vec![Ok(ChatResponse {
                    text: text.to_string(),
                    usage: Some(Usage::default()),
                    model: model.to_string(),
                })],
            )
        }

        fn err(name: &'static str, model: &str, err: LlmError) -> Self {
            Self::new(name, model, vec![Err(err)])
        }

        fn calls(&self) -> usize {
            self.calls.load(Ordering::SeqCst)
        }

        fn next_outcome(&self) -> LlmResult<ChatResponse> {
            self.calls.fetch_add(1, Ordering::SeqCst);
            let mut outcomes = self.outcomes.lock().unwrap();
            if outcomes.len() > 1 {
                outcomes.remove(0)
            } else {
                clone_outcome(&outcomes[0])
            }
        }

        fn record(&self, request: &ChatRequest, operation_id: Option<LlmOperationId>) {
            self.seen_requests.lock().unwrap().push(request.clone());
            self.seen_operation_ids.lock().unwrap().push(operation_id);
        }
    }

    fn clone_outcome(outcome: &LlmResult<ChatResponse>) -> LlmResult<ChatResponse> {
        match outcome {
            Ok(response) => Ok(response.clone()),
            Err(err) => Err(clone_err(err)),
        }
    }

    fn clone_err(err: &LlmError) -> LlmError {
        match err {
            LlmError::Provider {
                status,
                body,
                retry_after,
            } => LlmError::Provider {
                status: *status,
                body: body.clone(),
                retry_after: *retry_after,
            },
            LlmError::Serde(msg) => LlmError::Serde(msg.clone()),
            LlmError::UnexpectedShape(msg) => LlmError::UnexpectedShape(msg.clone()),
            LlmError::NotConfigured(msg) => LlmError::NotConfigured(msg.clone()),
            LlmError::Auth(msg) => LlmError::Auth(msg.clone()),
            LlmError::Schema(msg) => LlmError::Schema(msg.clone()),
            other => panic!("clone_err: unsupported variant in test double: {other}"),
        }
    }

    #[async_trait]
    impl LlmProvider for ScriptedLlm {
        fn name(&self) -> &'static str {
            self.name
        }

        fn model(&self) -> &str {
            &self.model
        }

        async fn complete(&self, request: ChatRequest) -> LlmResult<ChatResponse> {
            self.record(&request, None);
            self.next_outcome()
        }

        async fn complete_with_operation_id(
            &self,
            request: ChatRequest,
            operation_id: LlmOperationId,
        ) -> LlmResult<ChatResponse> {
            self.record(&request, Some(operation_id));
            self.next_outcome()
        }

        async fn complete_structured_raw(
            &self,
            request: ChatRequest,
            schema: serde_json::Value,
        ) -> LlmResult<serde_json::Value> {
            self.record(&request, None);
            self.seen_schemas.lock().unwrap().push(schema);
            self.next_outcome()
                .map(|response| serde_json::json!({ "text": response.text }))
        }

        async fn complete_structured_raw_with_operation_id(
            &self,
            request: ChatRequest,
            schema: serde_json::Value,
            operation_id: LlmOperationId,
        ) -> LlmResult<serde_json::Value> {
            self.record(&request, Some(operation_id));
            self.seen_schemas.lock().unwrap().push(schema);
            self.next_outcome()
                .map(|response| serde_json::json!({ "text": response.text }))
        }
    }

    fn transient(status: u16) -> LlmError {
        LlmError::Provider {
            status,
            body: "upstream body that must never leak".to_string(),
            retry_after: None,
        }
    }

    #[tokio::test]
    async fn chain_tries_candidates_in_order_and_stops_at_first_success() {
        let first = Arc::new(ScriptedLlm::err("anthropic", "claude", transient(503)));
        let second = Arc::new(ScriptedLlm::ok("gemini", "gemini-flash", "second answer"));
        let chain = FallbackLlmProvider::new(vec![
            Candidate::new("anthropic", "claude", first.clone()),
            Candidate::new("gemini", "gemini-flash", second.clone()),
        ]);

        let response = chain
            .complete(ChatRequest::user_prompt("hi"))
            .await
            .unwrap();

        assert_eq!(response.text, "second answer");
        assert_eq!(first.calls(), 1);
        assert_eq!(second.calls(), 1);
    }

    #[tokio::test]
    async fn all_four_methods_preserve_request_schema_and_operation_id() {
        let first = Arc::new(ScriptedLlm::err("primary", "m1", transient(500)));
        let second = Arc::new(ScriptedLlm::ok("secondary", "m2", "ok"));
        let chain = FallbackLlmProvider::new(vec![
            Candidate::new("primary", "m1", first.clone()),
            Candidate::new("secondary", "m2", second.clone()),
        ]);
        let request = ChatRequest::user_prompt("preserve me").with_max_tokens(42);
        let schema = serde_json::json!({ "type": "object" });
        let op_id = LlmOperationId::new();

        chain.complete(request.clone()).await.unwrap();
        chain
            .complete_with_operation_id(request.clone(), op_id)
            .await
            .unwrap();
        chain
            .complete_structured_raw(request.clone(), schema.clone())
            .await
            .unwrap();
        chain
            .complete_structured_raw_with_operation_id(request.clone(), schema.clone(), op_id)
            .await
            .unwrap();

        // Every call advanced past `first` (transient) onto `second`, which
        // must have seen the exact same request/schema/operation id on
        // each of the four entry points.
        assert_eq!(second.calls(), 4);
        let seen_requests = second.seen_requests.lock().unwrap();
        assert!(seen_requests.iter().all(
            |r| matches!(&r.messages[..], [ChatMessage { content, .. }] if content == "preserve me")
                && r.max_tokens == 42
        ));
        let seen_schemas = second.seen_schemas.lock().unwrap();
        assert_eq!(seen_schemas.len(), 2);
        assert!(seen_schemas.iter().all(|s| *s == schema));
        let seen_ops = second.seen_operation_ids.lock().unwrap();
        // complete/complete_structured_raw pass None; the *_with_operation_id
        // calls pass the same op_id every time.
        assert_eq!(*seen_ops, vec![None, Some(op_id), None, Some(op_id)]);
    }

    #[tokio::test]
    async fn transient_statuses_advance_to_the_next_candidate() {
        for status in [429, 500, 502, 503, 504] {
            let first = Arc::new(ScriptedLlm::err("primary", "m1", transient(status)));
            let second = Arc::new(ScriptedLlm::ok("secondary", "m2", "ok"));
            let chain = FallbackLlmProvider::new(vec![
                Candidate::new("primary", "m1", first.clone()),
                Candidate::new("secondary", "m2", second.clone()),
            ]);

            let result = chain.complete(ChatRequest::user_prompt("hi")).await;
            assert!(result.is_ok(), "status {status} should have advanced");
            assert_eq!(second.calls(), 1, "status {status} should have advanced");
        }
    }

    #[tokio::test]
    async fn deterministic_errors_stop_on_the_first_candidate() {
        let deterministic = [
            LlmError::Provider {
                status: 400,
                body: String::new(),
                retry_after: None,
            },
            LlmError::Provider {
                status: 401,
                body: String::new(),
                retry_after: None,
            },
            LlmError::Provider {
                status: 403,
                body: String::new(),
                retry_after: None,
            },
            LlmError::Provider {
                status: 404,
                body: String::new(),
                retry_after: None,
            },
            LlmError::Provider {
                status: 422,
                body: String::new(),
                retry_after: None,
            },
            LlmError::Schema("bad schema".into()),
            LlmError::Serde("deserialize failed".into()),
            LlmError::UnexpectedShape("no tool block".into()),
        ];
        for err in deterministic {
            let expected_class = err.class();
            let first = Arc::new(ScriptedLlm::err("primary", "m1", err));
            let second = Arc::new(ScriptedLlm::ok("secondary", "m2", "unreachable"));
            let chain = FallbackLlmProvider::new(vec![
                Candidate::new("primary", "m1", first.clone()),
                Candidate::new("secondary", "m2", second.clone()),
            ]);

            let result = chain.complete(ChatRequest::user_prompt("hi")).await;
            let returned = result.expect_err("deterministic error must propagate, not succeed");
            assert_eq!(returned.class(), expected_class);
            assert_eq!(first.calls(), 1);
            assert_eq!(
                second.calls(),
                0,
                "class {expected_class} must not try the next candidate"
            );
        }
    }

    #[tokio::test]
    async fn all_candidates_failing_returns_a_bounded_aggregate_error() {
        let first = Arc::new(ScriptedLlm::err(
            "primary",
            "m1",
            transient_with_body(500, "primary secret body"),
        ));
        let second = Arc::new(ScriptedLlm::err(
            "secondary",
            "m2",
            transient_with_body(503, "secondary secret body"),
        ));
        let chain = FallbackLlmProvider::new(vec![
            Candidate::new("primary", "m1", first.clone()),
            Candidate::new("secondary", "m2", second.clone()),
        ]);

        let err = chain
            .complete(ChatRequest::user_prompt("hi"))
            .await
            .expect_err("every candidate failed");
        match &err {
            LlmError::AllCandidatesFailed { attempted, summary } => {
                assert_eq!(*attempted, 2);
                assert!(summary.contains("primary/m1"));
                assert!(summary.contains("secondary/m2"));
                assert!(summary.contains("500"));
                assert!(summary.contains("503"));
                assert!(!summary.contains("secret body"));
            }
            other => panic!("expected AllCandidatesFailed, got {other}"),
        }
    }

    fn transient_with_body(status: u16, body: &str) -> LlmError {
        LlmError::Provider {
            status,
            body: body.to_string(),
            retry_after: None,
        }
    }

    #[test]
    fn provider_retry_hints_are_clamped_to_the_circuit_bounds() {
        let chain = FallbackLlmProvider::new(vec![Candidate::new(
            "primary",
            "m1",
            Arc::new(ScriptedLlm::ok("primary", "m1", "unused")),
        )]);

        for (hint, expected) in [(5, 30), (90, 90), (7_200, 3_600)] {
            let before = Instant::now();
            chain.record_error(
                0,
                &LlmError::Provider {
                    status: 429,
                    body: String::new(),
                    retry_after: Some(Duration::from_secs(hint)),
                },
            );
            let until = chain.entries[0]
                .state
                .lock()
                .unwrap()
                .circuit_open_until
                .unwrap();
            let actual = until.duration_since(before);
            assert!(
                actual >= Duration::from_secs(expected)
                    && actual < Duration::from_secs(expected + 1),
                "hint {hint}s produced {actual:?}"
            );
        }
    }

    #[tokio::test]
    async fn an_open_circuit_skips_only_its_candidate() {
        let first = Arc::new(ScriptedLlm::err("primary", "m1", transient(500)));
        let second = Arc::new(ScriptedLlm::ok("secondary", "m2", "ok"));
        let chain = FallbackLlmProvider::new(vec![
            Candidate::new("primary", "m1", first.clone()),
            Candidate::new("secondary", "m2", second.clone()),
        ]);

        // First call: primary fails transiently and opens its circuit;
        // secondary answers.
        chain
            .complete(ChatRequest::user_prompt("hi"))
            .await
            .unwrap();
        assert_eq!(first.calls(), 1);
        assert_eq!(second.calls(), 1);

        // Second call, still inside the cooldown: primary must be skipped
        // outright (not attempted again), secondary answers again.
        chain
            .complete(ChatRequest::user_prompt("hi"))
            .await
            .unwrap();
        assert_eq!(first.calls(), 1, "an open circuit must not be attempted");
        assert_eq!(second.calls(), 2);
    }

    #[test]
    fn a_success_closes_an_open_circuit_immediately() {
        let chain = FallbackLlmProvider::new(vec![Candidate::new(
            "primary",
            "m1",
            Arc::new(ScriptedLlm::ok("primary", "m1", "ok")),
        )]);
        // Simulate an open circuit far in the future — a real cooldown, not
        // yet elapsed.
        {
            let mut state = chain.entries[0].state.lock().unwrap();
            state.circuit_open_until = Some(Instant::now() + Duration::from_secs(600));
        }
        assert!(chain.circuit_open(0));

        chain.record_success(0);

        assert!(
            !chain.circuit_open(0),
            "a success must close the circuit immediately, not wait out the cooldown"
        );
    }

    #[test]
    fn an_elapsed_cooldown_closes_the_circuit_without_an_explicit_success() {
        let chain = FallbackLlmProvider::new(vec![Candidate::new(
            "primary",
            "m1",
            Arc::new(ScriptedLlm::ok("primary", "m1", "ok")),
        )]);
        {
            let mut state = chain.entries[0].state.lock().unwrap();
            state.circuit_open_until = Some(Instant::now() - Duration::from_secs(1));
        }
        assert!(!chain.circuit_open(0));
    }

    #[tokio::test]
    async fn candidate_health_reports_labels_last_selected_and_redacted_errors() {
        let first = Arc::new(ScriptedLlm::err(
            "primary",
            "m1",
            transient_with_body(500, "must not leak"),
        ));
        let second = Arc::new(ScriptedLlm::ok("secondary", "m2", "ok"));
        let chain = FallbackLlmProvider::new(vec![
            Candidate::new("primary", "m1", first),
            Candidate::new("secondary", "m2", second),
        ]);

        chain
            .complete(ChatRequest::user_prompt("hi"))
            .await
            .unwrap();

        let health = chain.candidate_health();
        assert_eq!(health.len(), 2);
        assert_eq!(health[0].provider, "primary");
        assert_eq!(health[0].model, "m1");
        assert!(!health[0].last_selected);
        assert_eq!(health[0].last_error_status, Some(500));
        assert_eq!(health[0].last_error_class.as_deref(), Some("provider"));
        assert!(health[0].circuit_open_until.is_some());
        assert!(health[1].last_selected);
        assert!(health[1].last_success_at.is_some());
        assert!(health[1].circuit_open_until.is_none());

        let rendered = format!("{health:?}");
        assert!(!rendered.contains("must not leak"));
    }

    /// Collects what a subscriber writes so a test can assert on log lines.
    #[derive(Clone, Default)]
    struct CapturedLog(Arc<Mutex<Vec<u8>>>);

    impl CapturedLog {
        fn text(&self) -> String {
            String::from_utf8_lossy(&self.0.lock().unwrap()).into_owned()
        }
    }

    impl std::io::Write for CapturedLog {
        fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
            self.0.lock().unwrap().extend_from_slice(buf);
            Ok(buf.len())
        }

        fn flush(&mut self) -> std::io::Result<()> {
            Ok(())
        }
    }

    impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for CapturedLog {
        type Writer = Self;

        fn make_writer(&'a self) -> Self::Writer {
            self.clone()
        }
    }

    fn info_subscriber(writer: CapturedLog) -> impl tracing::Subscriber + Send + Sync {
        tracing_subscriber::fmt()
            .with_writer(writer)
            .with_max_level(tracing::Level::INFO)
            .with_ansi(false)
            .finish()
    }

    /// Captures this thread's log lines. tracing-core caches each callsite's
    /// interest process-wide, so a parallel test reaching the same `warn!`
    /// with no subscriber can pin it to `never` and drop the line here. As in
    /// `ai-memory-mcp`'s `install_capture`, a second live `Dispatch` stops that
    /// and `rebuild_interest_cache()` heals a callsite already pinned. The
    /// companion must outlive the capture, so the caller holds it.
    fn capture_info_logs() -> (
        CapturedLog,
        tracing::subscriber::DefaultGuard,
        tracing::Dispatch,
    ) {
        let captured = CapturedLog::default();
        let companion = tracing::Dispatch::new(info_subscriber(CapturedLog::default()));
        let guard = tracing::subscriber::set_default(info_subscriber(captured.clone()));
        tracing::callsite::rebuild_interest_cache();
        (captured, guard, companion)
    }

    #[tokio::test]
    async fn a_transient_failure_logs_the_failing_candidate_without_its_body() {
        let (log, _guard, _companion) = capture_info_logs();
        let chain = FallbackLlmProvider::new(vec![
            Candidate::new(
                "openai-compat",
                "gemini-3.8-flash",
                Arc::new(ScriptedLlm::err(
                    "openai-compat",
                    "gemini-3.8-flash",
                    LlmError::Provider {
                        status: 429,
                        body: "quota body sk-secret-echo".into(),
                        retry_after: Some(Duration::from_secs(600)),
                    },
                )),
            ),
            Candidate::new(
                "openai-compat",
                "gemini-3.6-flash",
                Arc::new(ScriptedLlm::ok("openai-compat", "gemini-3.6-flash", "ok")),
            ),
        ]);

        chain
            .complete(ChatRequest::user_prompt("hi"))
            .await
            .unwrap();

        let text = log.text();
        let failure = text
            .lines()
            .find(|line| line.contains("skipping it until its cooldown ends"))
            .expect("the transient failure is logged");
        assert!(failure.contains("gemini-3.8-flash"), "{failure}");
        assert!(failure.contains("429"), "{failure}");
        assert!(failure.contains("cooldown_secs=600"), "{failure}");
        assert!(!text.contains("sk-secret-echo"), "the body leaked: {text}");
        assert!(
            text.lines()
                .any(|line| line.contains("now answering") && line.contains("gemini-3.6-flash")),
            "the switch to the fallback is logged: {text}"
        );
    }

    #[tokio::test]
    async fn the_answering_candidate_is_logged_only_when_it_changes() {
        let (log, _guard, _companion) = capture_info_logs();
        let chain = FallbackLlmProvider::new(vec![
            Candidate::new(
                "gemini",
                "primary",
                Arc::new(ScriptedLlm::err(
                    "gemini",
                    "primary",
                    LlmError::Provider {
                        status: 503,
                        body: "overloaded".into(),
                        retry_after: None,
                    },
                )),
            ),
            Candidate::new(
                "gemini",
                "spare",
                Arc::new(ScriptedLlm::ok("gemini", "spare", "ok")),
            ),
        ]);

        // The primary's circuit stays open across these calls, so each one is
        // answered by the spare: one switch, then silence.
        for _ in 0..3 {
            chain
                .complete(ChatRequest::user_prompt("hi"))
                .await
                .unwrap();
        }
        assert_eq!(log.text().matches("now answering").count(), 1);

        // Control: a chain whose primary answers never logs a switch.
        let (quiet_log, _quiet_guard, _quiet_companion) = capture_info_logs();
        let healthy = FallbackLlmProvider::new(vec![Candidate::new(
            "gemini",
            "primary",
            Arc::new(ScriptedLlm::ok("gemini", "primary", "ok")),
        )]);
        healthy
            .complete(ChatRequest::user_prompt("hi"))
            .await
            .unwrap();
        assert!(!quiet_log.text().contains("now answering"));
    }

    #[test]
    fn concurrent_transient_failures_log_only_the_circuit_transition() {
        let (log, _guard, _companion) = capture_info_logs();
        let chain = FallbackLlmProvider::new(vec![Candidate::new(
            "gemini",
            "primary",
            Arc::new(ScriptedLlm::ok("gemini", "primary", "unused")),
        )]);
        let failure = LlmError::Provider {
            status: 503,
            body: "must not leak".into(),
            retry_after: Some(Duration::from_secs(600)),
        };

        // Two requests can both reach the provider before the first failure
        // opens its circuit. The later result still refreshes health state,
        // but it must not repeat the outage warning.
        chain.record_error(0, &failure);
        chain.record_error(0, &failure);

        let text = log.text();
        assert_eq!(
            text.matches("skipping it until its cooldown ends").count(),
            1,
            "an already-open circuit repeated its warning: {text}"
        );
        assert!(!text.contains("must not leak"), "the body leaked: {text}");
    }
}