Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
feat(llm): log fallback chain candidate failures and switches
Renato Regis7fd426d144f4
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex a26bbb89..098aec97 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -73,6 +73,14 @@ version in [`docs/changelog/`](docs/changelog/):checkpoints, compactions, and annotations remain inline; the completesanitized tool evidence stays searchable in the ledger (#1607).+- The LLM fallback chain now logs its own events. A transient candidate+ failure logs a warning with that candidate's provider, model, chain+ position, error class, HTTP status and cooldown, and an info line records+ each change of the answering candidate. Before, the chain logged nothing,+ and callers such as the reranker named only the first candidate, so a log+ could blame the primary for a timeout another candidate caused. Response+ bodies are never logged (#1628).+### Fixed- `ai-memory user list` and `ai-memory api-key list` now size their columnsdiff --git a/Cargo.lock b/Cargo.lockindex 31e682e3..8bb94904 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -210,6 +210,7 @@ dependencies = ["tokenizers","tokio","tracing",+ "tracing-subscriber","uuid","wiremock",]diff --git a/crates/ai-memory-llm/Cargo.toml b/crates/ai-memory-llm/Cargo.tomlindex fe16c791..a14d433a 100644--- a/crates/ai-memory-llm/Cargo.toml+++ b/crates/ai-memory-llm/Cargo.toml@@ -54,6 +54,8 @@ rstest.workspace = true# In-process HTTP mock for the openai-compat strict-fallback tests.# Scoped to dev-dependencies so production binaries don't pull it in.wiremock = "0.6"+# Captures the fallback chain's log lines in its unit tests.+tracing-subscriber.workspace = true[lints]workspace = truediff --git a/crates/ai-memory-llm/src/fallback.rs b/crates/ai-memory-llm/src/fallback.rsindex a433781a..e5c9858f 100644--- a/crates/ai-memory-llm/src/fallback.rs+++ b/crates/ai-memory-llm/src/fallback.rs@@ -16,6 +16,7 @@ use std::time::{Duration, Instant};use async_trait::async_trait;use jiff::Timestamp;+use tracing::{debug, info, warn};use crate::error::{LlmError, LlmResult};use crate::health::CandidateHealth;@@ -118,24 +119,70 @@ impl FallbackLlmProvider {state.last_success_at = Some(Timestamp::now());state.circuit_open_until = None;}- *self.last_selected.lock().unwrap_or_else(|e| e.into_inner()) = Some(index);+ let previous = self+ .last_selected+ .lock()+ .unwrap_or_else(|e| e.into_inner())+ .replace(index);+ // Logged on a change of answering candidate only, so a primary that+ // stays down for an hour yields one line instead of one per call.+ let switched = match previous {+ Some(previous) => previous != index,+ None => index > 0,+ };+ if switched {+ let candidate = &self.entries[index].candidate;+ info!(+ provider = candidate.provider,+ model = %candidate.model,+ position = index,+ "llm fallback chain now answering from this candidate"+ );+ }}fn record_error(&self, index: usize, err: &LlmError) {- let mut state = self.entries[index]- .state- .lock()- .unwrap_or_else(|e| e.into_inner());- state.last_error_at = Some(Timestamp::now());- state.last_error_status = err.http_status();- state.last_error_class = Some(err.class());- if err.is_transient() {- let cooldown = err- .retry_after()+ let cooldown = err.is_transient().then(|| {+ err.retry_after().unwrap_or(CIRCUIT_COOLDOWN).max(CIRCUIT_COOLDOWN)- .min(MAX_CIRCUIT_COOLDOWN);- state.circuit_open_until = Some(Instant::now() + cooldown);+ .min(MAX_CIRCUIT_COOLDOWN)+ });+ {+ let mut state = self.entries[index]+ .state+ .lock()+ .unwrap_or_else(|e| e.into_inner());+ state.last_error_at = Some(Timestamp::now());+ state.last_error_status = err.http_status();+ state.last_error_class = Some(err.class());+ if let Some(cooldown) = cooldown {+ state.circuit_open_until = Some(Instant::now() + cooldown);+ }+ }+ // Redacted fields only, as the reranker does: the `Display` of a+ // provider failure carries the upstream response body. The model is+ // the failing candidate's, which the caller cannot see: every caller+ // labels the chain with its first candidate.+ let candidate = &self.entries[index].candidate;+ match cooldown {+ Some(cooldown) => warn!(+ provider = candidate.provider,+ model = %candidate.model,+ position = index,+ error_class = err.class(),+ error_status = ?err.http_status(),+ cooldown_secs = cooldown.as_secs(),+ "llm fallback candidate failed; skipping it until its cooldown ends"+ ),+ None => debug!(+ provider = candidate.provider,+ model = %candidate.model,+ position = index,+ error_class = err.class(),+ error_status = ?err.http_status(),+ "llm fallback candidate failed deterministically; the chain stops here"+ ),}}@@ -801,4 +848,152 @@ mod tests {let rendered = format!("{health:?}");assert!(!rendered.contains("must not leak"));}++ /// Collects what a subscriber writes so a test can assert on log lines.+ #[derive(Clone, Default)]+ struct CapturedLog(Arc<Mutex<Vec<u8>>>);++ impl CapturedLog {+ fn text(&self) -> String {+ String::from_utf8_lossy(&self.0.lock().unwrap()).into_owned()+ }+ }++ impl std::io::Write for CapturedLog {+ fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {+ self.0.lock().unwrap().extend_from_slice(buf);+ Ok(buf.len())+ }++ fn flush(&mut self) -> std::io::Result<()> {+ Ok(())+ }+ }++ impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for CapturedLog {+ type Writer = Self;++ fn make_writer(&'a self) -> Self::Writer {+ self.clone()+ }+ }++ fn info_subscriber(writer: CapturedLog) -> impl tracing::Subscriber + Send + Sync {+ tracing_subscriber::fmt()+ .with_writer(writer)+ .with_max_level(tracing::Level::INFO)+ .with_ansi(false)+ .finish()+ }++ /// Captures this thread's log lines. tracing-core caches each callsite's+ /// interest process-wide, so a parallel test reaching the same `warn!`+ /// with no subscriber can pin it to `never` and drop the line here. As in+ /// `ai-memory-mcp`'s `install_capture`, a second live `Dispatch` stops that+ /// and `rebuild_interest_cache()` heals a callsite already pinned. The+ /// companion must outlive the capture, so the caller holds it.+ fn capture_info_logs() -> (+ CapturedLog,+ tracing::subscriber::DefaultGuard,+ tracing::Dispatch,+ ) {+ let captured = CapturedLog::default();+ let companion = tracing::Dispatch::new(info_subscriber(CapturedLog::default()));+ let guard = tracing::subscriber::set_default(info_subscriber(captured.clone()));+ tracing::callsite::rebuild_interest_cache();+ (captured, guard, companion)+ }++ #[tokio::test]+ async fn a_transient_failure_logs_the_failing_candidate_without_its_body() {+ let (log, _guard, _companion) = capture_info_logs();+ let chain = FallbackLlmProvider::new(vec![+ Candidate::new(+ "openai-compat",+ "gemini-3.8-flash",+ Arc::new(ScriptedLlm::err(+ "openai-compat",+ "gemini-3.8-flash",+ LlmError::Provider {+ status: 429,+ body: "quota body sk-secret-echo".into(),+ retry_after: Some(Duration::from_secs(600)),+ },+ )),+ ),+ Candidate::new(+ "openai-compat",+ "gemini-3.6-flash",+ Arc::new(ScriptedLlm::ok("openai-compat", "gemini-3.6-flash", "ok")),+ ),+ ]);++ chain+ .complete(ChatRequest::user_prompt("hi"))+ .await+ .unwrap();++ let text = log.text();+ let failure = text+ .lines()+ .find(|line| line.contains("skipping it until its cooldown ends"))+ .expect("the transient failure is logged");+ assert!(failure.contains("gemini-3.8-flash"), "{failure}");+ assert!(failure.contains("429"), "{failure}");+ assert!(failure.contains("cooldown_secs=600"), "{failure}");+ assert!(!text.contains("sk-secret-echo"), "the body leaked: {text}");+ assert!(+ text.lines()+ .any(|line| line.contains("now answering") && line.contains("gemini-3.6-flash")),+ "the switch to the fallback is logged: {text}"+ );+ }++ #[tokio::test]+ async fn the_answering_candidate_is_logged_only_when_it_changes() {+ let (log, _guard, _companion) = capture_info_logs();+ let chain = FallbackLlmProvider::new(vec![+ Candidate::new(+ "gemini",+ "primary",+ Arc::new(ScriptedLlm::err(+ "gemini",+ "primary",+ LlmError::Provider {+ status: 503,+ body: "overloaded".into(),+ retry_after: None,+ },+ )),+ ),+ Candidate::new(+ "gemini",+ "spare",+ Arc::new(ScriptedLlm::ok("gemini", "spare", "ok")),+ ),+ ]);++ // The primary's circuit stays open across these calls, so each one is+ // answered by the spare: one switch, then silence.+ for _ in 0..3 {+ chain+ .complete(ChatRequest::user_prompt("hi"))+ .await+ .unwrap();+ }+ assert_eq!(log.text().matches("now answering").count(), 1);++ // Control: a chain whose primary answers never logs a switch.+ let (quiet_log, _quiet_guard, _quiet_companion) = capture_info_logs();+ let healthy = FallbackLlmProvider::new(vec![Candidate::new(+ "gemini",+ "primary",+ Arc::new(ScriptedLlm::ok("gemini", "primary", "ok")),+ )]);+ healthy+ .complete(ChatRequest::user_prompt("hi"))+ .await+ .unwrap();+ assert!(!quiet_log.text().contains("now answering"));+ }}diff --git a/docs/llm-provider-fallback.md b/docs/llm-provider-fallback.mdindex d2ae0cde..c2982287 100644--- a/docs/llm-provider-fallback.md+++ b/docs/llm-provider-fallback.md@@ -146,6 +146,16 @@ provider/model fields for compatibility. Per-candidate state contains:Status records only calls the server already made. It does not probe fallbacksor trigger background recovery traffic.+The server log carries the same events as they happen, with the same redacted+fields and never a response body. A transient failure logs a warning naming+the failing candidate's provider, model, chain position, error class, HTTP+status and cooldown; that is the only place the failing model shows up,+because callers such as the reranker label the chain with its first+candidate. A deterministic failure logs at debug level, since the caller+reports the returned error itself. An info line records each change of the+candidate that answers, so a primary that stays down yields one line, not one+per call.+## Tests1. Unit-test chain order with fake `LlmProvider` instances.