Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

feat(llm): log fallback chain candidate failures and switches

Renato Regis7fd426d144f4
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a26bbb89..098aec97 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -73,6 +73,14 @@ version in [`docs/changelog/`](docs/changelog/):
checkpoints, compactions, and annotations remain inline; the complete
sanitized tool evidence stays searchable in the ledger (#1607).
+- The LLM fallback chain now logs its own events. A transient candidate
+ failure logs a warning with that candidate's provider, model, chain
+ position, error class, HTTP status and cooldown, and an info line records
+ each change of the answering candidate. Before, the chain logged nothing,
+ and callers such as the reranker named only the first candidate, so a log
+ could blame the primary for a timeout another candidate caused. Response
+ bodies are never logged (#1628).
+
### Fixed
- `ai-memory user list` and `ai-memory api-key list` now size their columns
diff --git a/Cargo.lock b/Cargo.lock
index 31e682e3..8bb94904 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -210,6 +210,7 @@ dependencies = [
"tokenizers",
"tokio",
"tracing",
+ "tracing-subscriber",
"uuid",
"wiremock",
]
diff --git a/crates/ai-memory-llm/Cargo.toml b/crates/ai-memory-llm/Cargo.toml
index fe16c791..a14d433a 100644
--- a/crates/ai-memory-llm/Cargo.toml
+++ b/crates/ai-memory-llm/Cargo.toml
@@ -54,6 +54,8 @@ rstest.workspace = true
# In-process HTTP mock for the openai-compat strict-fallback tests.
# Scoped to dev-dependencies so production binaries don't pull it in.
wiremock = "0.6"
+# Captures the fallback chain's log lines in its unit tests.
+tracing-subscriber.workspace = true
[lints]
workspace = true
diff --git a/crates/ai-memory-llm/src/fallback.rs b/crates/ai-memory-llm/src/fallback.rs
index a433781a..e5c9858f 100644
--- a/crates/ai-memory-llm/src/fallback.rs
+++ b/crates/ai-memory-llm/src/fallback.rs
@@ -16,6 +16,7 @@ use std::time::{Duration, Instant};
use async_trait::async_trait;
use jiff::Timestamp;
+use tracing::{debug, info, warn};
use crate::error::{LlmError, LlmResult};
use crate::health::CandidateHealth;
@@ -118,24 +119,70 @@ impl FallbackLlmProvider {
state.last_success_at = Some(Timestamp::now());
state.circuit_open_until = None;
}
- *self.last_selected.lock().unwrap_or_else(|e| e.into_inner()) = Some(index);
+ let previous = self
+ .last_selected
+ .lock()
+ .unwrap_or_else(|e| e.into_inner())
+ .replace(index);
+ // Logged on a change of answering candidate only, so a primary that
+ // stays down for an hour yields one line instead of one per call.
+ let switched = match previous {
+ Some(previous) => previous != index,
+ None => index > 0,
+ };
+ if switched {
+ let candidate = &self.entries[index].candidate;
+ info!(
+ provider = candidate.provider,
+ model = %candidate.model,
+ position = index,
+ "llm fallback chain now answering from this candidate"
+ );
+ }
}
fn record_error(&self, index: usize, err: &LlmError) {
- let mut state = self.entries[index]
- .state
- .lock()
- .unwrap_or_else(|e| e.into_inner());
- state.last_error_at = Some(Timestamp::now());
- state.last_error_status = err.http_status();
- state.last_error_class = Some(err.class());
- if err.is_transient() {
- let cooldown = err
- .retry_after()
+ let cooldown = err.is_transient().then(|| {
+ err.retry_after()
.unwrap_or(CIRCUIT_COOLDOWN)
.max(CIRCUIT_COOLDOWN)
- .min(MAX_CIRCUIT_COOLDOWN);
- state.circuit_open_until = Some(Instant::now() + cooldown);
+ .min(MAX_CIRCUIT_COOLDOWN)
+ });
+ {
+ let mut state = self.entries[index]
+ .state
+ .lock()
+ .unwrap_or_else(|e| e.into_inner());
+ state.last_error_at = Some(Timestamp::now());
+ state.last_error_status = err.http_status();
+ state.last_error_class = Some(err.class());
+ if let Some(cooldown) = cooldown {
+ state.circuit_open_until = Some(Instant::now() + cooldown);
+ }
+ }
+ // Redacted fields only, as the reranker does: the `Display` of a
+ // provider failure carries the upstream response body. The model is
+ // the failing candidate's, which the caller cannot see: every caller
+ // labels the chain with its first candidate.
+ let candidate = &self.entries[index].candidate;
+ match cooldown {
+ Some(cooldown) => warn!(
+ provider = candidate.provider,
+ model = %candidate.model,
+ position = index,
+ error_class = err.class(),
+ error_status = ?err.http_status(),
+ cooldown_secs = cooldown.as_secs(),
+ "llm fallback candidate failed; skipping it until its cooldown ends"
+ ),
+ None => debug!(
+ provider = candidate.provider,
+ model = %candidate.model,
+ position = index,
+ error_class = err.class(),
+ error_status = ?err.http_status(),
+ "llm fallback candidate failed deterministically; the chain stops here"
+ ),
}
}
@@ -801,4 +848,152 @@ mod tests {
let rendered = format!("{health:?}");
assert!(!rendered.contains("must not leak"));
}
+
+ /// Collects what a subscriber writes so a test can assert on log lines.
+ #[derive(Clone, Default)]
+ struct CapturedLog(Arc<Mutex<Vec<u8>>>);
+
+ impl CapturedLog {
+ fn text(&self) -> String {
+ String::from_utf8_lossy(&self.0.lock().unwrap()).into_owned()
+ }
+ }
+
+ impl std::io::Write for CapturedLog {
+ fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+ self.0.lock().unwrap().extend_from_slice(buf);
+ Ok(buf.len())
+ }
+
+ fn flush(&mut self) -> std::io::Result<()> {
+ Ok(())
+ }
+ }
+
+ impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for CapturedLog {
+ type Writer = Self;
+
+ fn make_writer(&'a self) -> Self::Writer {
+ self.clone()
+ }
+ }
+
+ fn info_subscriber(writer: CapturedLog) -> impl tracing::Subscriber + Send + Sync {
+ tracing_subscriber::fmt()
+ .with_writer(writer)
+ .with_max_level(tracing::Level::INFO)
+ .with_ansi(false)
+ .finish()
+ }
+
+ /// Captures this thread's log lines. tracing-core caches each callsite's
+ /// interest process-wide, so a parallel test reaching the same `warn!`
+ /// with no subscriber can pin it to `never` and drop the line here. As in
+ /// `ai-memory-mcp`'s `install_capture`, a second live `Dispatch` stops that
+ /// and `rebuild_interest_cache()` heals a callsite already pinned. The
+ /// companion must outlive the capture, so the caller holds it.
+ fn capture_info_logs() -> (
+ CapturedLog,
+ tracing::subscriber::DefaultGuard,
+ tracing::Dispatch,
+ ) {
+ let captured = CapturedLog::default();
+ let companion = tracing::Dispatch::new(info_subscriber(CapturedLog::default()));
+ let guard = tracing::subscriber::set_default(info_subscriber(captured.clone()));
+ tracing::callsite::rebuild_interest_cache();
+ (captured, guard, companion)
+ }
+
+ #[tokio::test]
+ async fn a_transient_failure_logs_the_failing_candidate_without_its_body() {
+ let (log, _guard, _companion) = capture_info_logs();
+ let chain = FallbackLlmProvider::new(vec![
+ Candidate::new(
+ "openai-compat",
+ "gemini-3.8-flash",
+ Arc::new(ScriptedLlm::err(
+ "openai-compat",
+ "gemini-3.8-flash",
+ LlmError::Provider {
+ status: 429,
+ body: "quota body sk-secret-echo".into(),
+ retry_after: Some(Duration::from_secs(600)),
+ },
+ )),
+ ),
+ Candidate::new(
+ "openai-compat",
+ "gemini-3.6-flash",
+ Arc::new(ScriptedLlm::ok("openai-compat", "gemini-3.6-flash", "ok")),
+ ),
+ ]);
+
+ chain
+ .complete(ChatRequest::user_prompt("hi"))
+ .await
+ .unwrap();
+
+ let text = log.text();
+ let failure = text
+ .lines()
+ .find(|line| line.contains("skipping it until its cooldown ends"))
+ .expect("the transient failure is logged");
+ assert!(failure.contains("gemini-3.8-flash"), "{failure}");
+ assert!(failure.contains("429"), "{failure}");
+ assert!(failure.contains("cooldown_secs=600"), "{failure}");
+ assert!(!text.contains("sk-secret-echo"), "the body leaked: {text}");
+ assert!(
+ text.lines()
+ .any(|line| line.contains("now answering") && line.contains("gemini-3.6-flash")),
+ "the switch to the fallback is logged: {text}"
+ );
+ }
+
+ #[tokio::test]
+ async fn the_answering_candidate_is_logged_only_when_it_changes() {
+ let (log, _guard, _companion) = capture_info_logs();
+ let chain = FallbackLlmProvider::new(vec![
+ Candidate::new(
+ "gemini",
+ "primary",
+ Arc::new(ScriptedLlm::err(
+ "gemini",
+ "primary",
+ LlmError::Provider {
+ status: 503,
+ body: "overloaded".into(),
+ retry_after: None,
+ },
+ )),
+ ),
+ Candidate::new(
+ "gemini",
+ "spare",
+ Arc::new(ScriptedLlm::ok("gemini", "spare", "ok")),
+ ),
+ ]);
+
+ // The primary's circuit stays open across these calls, so each one is
+ // answered by the spare: one switch, then silence.
+ for _ in 0..3 {
+ chain
+ .complete(ChatRequest::user_prompt("hi"))
+ .await
+ .unwrap();
+ }
+ assert_eq!(log.text().matches("now answering").count(), 1);
+
+ // Control: a chain whose primary answers never logs a switch.
+ let (quiet_log, _quiet_guard, _quiet_companion) = capture_info_logs();
+ let healthy = FallbackLlmProvider::new(vec![Candidate::new(
+ "gemini",
+ "primary",
+ Arc::new(ScriptedLlm::ok("gemini", "primary", "ok")),
+ )]);
+ healthy
+ .complete(ChatRequest::user_prompt("hi"))
+ .await
+ .unwrap();
+ assert!(!quiet_log.text().contains("now answering"));
+ }
}
diff --git a/docs/llm-provider-fallback.md b/docs/llm-provider-fallback.md
index d2ae0cde..c2982287 100644
--- a/docs/llm-provider-fallback.md
+++ b/docs/llm-provider-fallback.md
@@ -146,6 +146,16 @@ provider/model fields for compatibility. Per-candidate state contains:
Status records only calls the server already made. It does not probe fallbacks
or trigger background recovery traffic.
+The server log carries the same events as they happen, with the same redacted
+fields and never a response body. A transient failure logs a warning naming
+the failing candidate's provider, model, chain position, error class, HTTP
+status and cooldown; that is the only place the failing model shows up,
+because callers such as the reranker label the chain with its first
+candidate. A deterministic failure logs at debug level, since the caller
+reports the returned error itself. An info line records each change of the
+candidate that answers, so a primary that stays down yields one line, not one
+per call.
+
## Tests
1. Unit-test chain order with fake `LlmProvider` instances.