Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
feat(cli): probe the server's own provider with llm-test --server
AkitaOnRails3ba144189c58
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex e618d1ab..be2b06e6 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -13,6 +13,15 @@ version in [`docs/changelog/`](docs/changelog/):### Added+- `ai-memory llm-test --server` probes the server's own configured provider+ instead of building one from the caller's environment, through a new+ root-only `POST /admin/llm-test`. The retry hint that `status` and the health+ panel print now names it: with the server in a container or on another host+ its keys exist only there, so the old client-side hint failed with "no API+ key" or a provider `400` while the server was healthy. The probe runs one+ fixed short prompt, reports provider, model, latency and the error class, and+ scrubs the provider's message before printing it (#1639).+- Added `[handoff] deliver_on_session_start = false` to `.ai-memory.toml`. Aproject that sets it no longer gets its handoff, or the continuity notice, atsession start; the handoff is still created when the session ends and staysdiff --git a/crates/ai-memory-cli/src/cli.rs b/crates/ai-memory-cli/src/cli.rsindex 3fe35da2..eee5f01f 100644--- a/crates/ai-memory-cli/src/cli.rs+++ b/crates/ai-memory-cli/src/cli.rs@@ -3197,15 +3197,29 @@ pub struct PendingWriteRejectArgs {/// Arguments for `llm-test`.#[derive(Debug, Args)]pub struct LlmTestArgs {+ /// Probe the server's own configured provider instead of building one+ /// from this shell's environment.+ ///+ /// The retry hint the server and `/web` print uses this form: a server in+ /// a container or on another host holds the keys, so a client-side+ /// `llm-test` fails while the server is healthy (#1639). Takes no+ /// `--provider`/`--model`/`--prompt`/`--base-url`/`--api-key`: the probe+ /// runs what the server is configured with, and a client cannot point it+ /// at another URL or hand it a key.+ #[arg(+ long,+ conflicts_with_all = ["provider", "model", "prompt", "structured", "base_url", "api_key"]+ )]+ pub server: bool,/// Provider to test.- #[arg(long, value_enum)]- pub provider: LlmProviderChoice,+ #[arg(long, value_enum, required_unless_present = "server")]+ pub provider: Option<LlmProviderChoice>,/// Model identifier (e.g. `claude-haiku-4-5`, `gpt-5.4-mini`, `llama3.1:8b`).- #[arg(long)]- pub model: String,+ #[arg(long, required_unless_present = "server")]+ pub model: Option<String>,/// Prompt to send.- #[arg(long)]- pub prompt: String,+ #[arg(long, required_unless_present = "server")]+ pub prompt: Option<String>,/// Request a small JSON-schema response instead of plain text.#[arg(long)]pub structured: bool,@@ -5566,9 +5580,13 @@ mod tests {let Command::LlmTest(args) = cli.command else {panic!("expected llm-test command");};- assert!(matches!(args.provider, LlmProviderChoice::AnthropicOauth));- assert_eq!(args.model, "claude-sonnet-4-6");- assert_eq!(args.prompt, "ping");+ assert!(matches!(+ args.provider,+ Some(LlmProviderChoice::AnthropicOauth)+ ));+ assert_eq!(args.model.as_deref(), Some("claude-sonnet-4-6"));+ assert_eq!(args.prompt.as_deref(), Some("ping"));+ assert!(!args.server);}#[test]@@ -5589,10 +5607,43 @@ mod tests {let Command::LlmTest(args) = cli.command else {panic!("expected llm-test command");};- assert!(matches!(args.provider, LlmProviderChoice::Codex));+ assert!(matches!(args.provider, Some(LlmProviderChoice::Codex)));assert!(args.structured);}+ /// `--server` runs the probe on the server, so none of the client-side+ /// provider flags apply and passing one is a parse error rather than a+ /// silently ignored key (#1639).+ #[test]+ fn llm_test_server_takes_no_client_side_provider_flags() {+ let cli = Cli::try_parse_from(["ai-memory", "llm-test", "--server"]).unwrap();+ let Command::LlmTest(args) = cli.command else {+ panic!("expected llm-test command");+ };+ assert!(args.server);+ assert!(args.provider.is_none() && args.model.is_none() && args.prompt.is_none());++ for flag in [+ "--provider",+ "--model",+ "--prompt",+ "--api-key",+ "--base-url",+ ] {+ let mut argv = vec!["ai-memory", "llm-test", "--server", flag];+ let value = match flag {+ "--provider" => "anthropic",+ "--api-key" => "sk-x",+ _ => "x",+ };+ argv.push(value);+ assert!(+ Cli::try_parse_from(&argv).is_err(),+ "`{flag}` must conflict with --server"+ );+ }+ }+#[test]fn completions_parses_every_supported_shell() {for (arg, expected) in [diff --git a/crates/ai-memory-cli/src/commands/llm_test.rs b/crates/ai-memory-cli/src/commands/llm_test.rsindex b82d1a91..0d7d5b11 100644--- a/crates/ai-memory-cli/src/commands/llm_test.rs+++ b/crates/ai-memory-cli/src/commands/llm_test.rs@@ -6,6 +6,7 @@ use tracing::info;use crate::cli::{LlmProviderChoice, LlmTestArgs};use crate::config::Config;+use crate::http_client::{ServerEndpoint, post_json};/// Run the `llm-test` subcommand.///@@ -13,14 +14,22 @@ use crate::config::Config;/// Returns an error if the provider cannot be constructed, the env/// lacks the required keys, or the HTTP call fails.pub async fn run(config: &Config, args: LlmTestArgs) -> Result<()> {- let provider = ProviderChoice::from(args.provider);+ if args.server {+ return run_against_server(config).await;+ }+ let provider = ProviderChoice::from(+ args.provider+ .context("--provider is required unless --server is passed")?,+ );let api_key_override = args.api_key.filter(|s| !s.is_empty()).map(secrecy::SecretString::from);let provider_config = ProviderConfig {provider,- model: args.model,+ model: args+ .model+ .context("--model is required unless --server is passed")?,auth: config.provider_auth(provider, api_key_override),base_url: args.base_url.or_else(|| config.llm_test_base_url(provider)),compat_strict: config.llm_compat_strict,@@ -37,7 +46,10 @@ pub async fn run(config: &Config, args: LlmTestArgs) -> Result<()> {model = client.model(),"sending prompt",);- let request = representative_request(args.prompt);+ let request = representative_request(+ args.prompt+ .context("--prompt is required unless --server is passed")?,+ );if args.structured {let value = client.complete_structured_raw(@@ -115,6 +127,22 @@ impl From<LlmProviderChoice> for ProviderChoice {}}+/// Probe the server's own configured provider (`--server`).+///+/// The server runs the probe with a fixed short prompt against what its+/// configuration holds, so this works from a machine that has no keys at all —+/// which is the whole point (#1639).+async fn run_against_server(config: &Config) -> Result<()> {+ let endpoint = ServerEndpoint::from_config_resolving_auth(config).await;+ let report: serde_json::Value = post_json(&endpoint, "/admin/llm-test", &()).await?;+ let provider = report["provider"].as_str().unwrap_or("(unknown)");+ let model = report["model"].as_str().unwrap_or("(unknown)");+ let latency = report["latency_ms"].as_u64().unwrap_or(0);+ println!("--- server provider: {provider} / {model} ({latency} ms) ---");+ println!("{}", serde_json::to_string_pretty(&report)?);+ Ok(())+}+#[cfg(test)]mod tests {use super::*;diff --git a/crates/ai-memory-cli/src/commands/serve.rs b/crates/ai-memory-cli/src/commands/serve.rsindex 07d81786..91a4756e 100644--- a/crates/ai-memory-cli/src/commands/serve.rs+++ b/crates/ai-memory-cli/src/commands/serve.rs@@ -3152,7 +3152,7 @@ fn configure_consolidator(};let provider_name = cfg.provider.name().to_string();let model = cfg.model.clone();- let retry_hint = llm_retry_hint(&provider_name, &model, cfg.base_url.as_deref());+ let retry_hint = llm_retry_hint(&provider_name, &model);// Routes through the same construction `llm_provider_config` just// confirmed is `Some`: wraps `[primary, fallbacks...]` in a// `FallbackLlmProvider` when `llm_fallbacks` is non-empty, else returns@@ -3304,13 +3304,16 @@ fn validate_web_ui_args(enable_web: bool, web_ui_dir: Option<&Path>) -> Result<(Ok(())}-fn llm_retry_hint(provider: &str, model: &str, base_url: Option<&str>) -> String {- let mut command = format!("ai-memory llm-test --provider {provider} --model {model}");- if let Some(base_url) = base_url {- command.push_str(&format!(" --base-url {base_url}"));- }- command.push_str(" --prompt ping");- command+/// What to run when the server's provider call failed.+///+/// The probe runs *on the server* (`--server`): a client-side `llm-test`+/// builds its provider from the caller's environment, and when the server runs+/// in a container or on another host the keys exist only there, so the hint+/// answered "no API key" or a provider 400 while the server was healthy+/// (#1639). `provider`/`model` are still named, so the report is readable+/// without opening the config.+fn llm_retry_hint(provider: &str, model: &str) -> String {+ format!("ai-memory llm-test --server # server provider: {provider} / {model}")}/// Liveness probe for process supervisors.@@ -3627,6 +3630,26 @@ fn host_without_port(host: &str) -> &str {#[cfg(test)]mod tests {+ /// The retry hint must name the probe that runs where the keys are (#1639):+ /// a client-side `llm-test` builds its provider from the caller's+ /// environment, which is the wrong one for a container or a remote server.+ #[test]+ fn the_retry_hint_probes_the_server_not_the_clients_environment() {+ let hint = super::llm_retry_hint("anthropic", "claude-haiku-4-5");+ assert!(+ hint.starts_with("ai-memory llm-test --server"),+ "the hint must run the probe on the server: {hint}"+ );+ assert!(+ !hint.contains("--provider") && !hint.contains("--api-key"),+ "the server-side probe takes no client-side provider flags: {hint}"+ );+ assert!(+ hint.contains("anthropic") && hint.contains("claude-haiku-4-5"),+ "the hint must still name what failed: {hint}"+ );+ }+#[cfg(feature = "management-ui")]#[tokio::test]async fn management_ui_rejects_invalid_mounts_before_opening_the_store() {diff --git a/crates/ai-memory-llm/src/health.rs b/crates/ai-memory-llm/src/health.rsindex 5642bf3d..c63cb2ca 100644--- a/crates/ai-memory-llm/src/health.rs+++ b/crates/ai-memory-llm/src/health.rs@@ -749,7 +749,13 @@ impl Embedder for HealthRecordingEmbedder {}}-fn error_status_and_message(err: &LlmError) -> (Option<u16>, String) {+/// The HTTP status and a bounded, redacted message for `err`.+///+/// The admin `llm-test` route answers with this so a server-side probe reports+/// the same error class the health snapshot records, never the provider's full+/// body (#1639).+#[must_use]+pub fn error_status_and_message(err: &LlmError) -> (Option<u16>, String) {match err {LlmError::Http(e) => (e.status().map(|status| status.as_u16()),diff --git a/crates/ai-memory-llm/src/lib.rs b/crates/ai-memory-llm/src/lib.rsindex 47c652c6..fdcede9d 100644--- a/crates/ai-memory-llm/src/lib.rs+++ b/crates/ai-memory-llm/src/lib.rs@@ -133,7 +133,7 @@ pub use gemini::GeminiProvider;pub use google::{DEFAULT_MODEL as GOOGLE_DEFAULT_EMBED_MODEL, GoogleEmbedder};pub use health::{CandidateHealth, LlmWorkloadHealthSnapshot, ProviderHealth, ProviderHealthSnapshot,- ProviderHealthStatus, ProviderRoleHealthSnapshot,+ ProviderHealthStatus, ProviderRoleHealthSnapshot, error_status_and_message,};#[cfg(feature = "local-embeddings")]pub use local::{LOCAL_DIM, LOCAL_MODEL, LocalEmbedder, fetch_model, model_present};diff --git a/crates/ai-memory-mcp/src/admin.rs b/crates/ai-memory-mcp/src/admin.rsindex e826d0f6..d8de65bc 100644--- a/crates/ai-memory-mcp/src/admin.rs+++ b/crates/ai-memory-mcp/src/admin.rs@@ -71,7 +71,10 @@ use ai_memory_core::{ActiveProject, AgentKind, AutoImproveProposalId, Capability, DEFAULT_PROJECT_NAME,DEFAULT_WORKSPACE_NAME, PagePath, ProjectId, SessionId, Tier, WorkspaceId,};-use ai_memory_llm::{Embedder, LlmProvider, ProviderHealth, ProviderHealthSnapshot};+use ai_memory_llm::{+ ChatRequest, Embedder, LlmProvider, ProviderHealth, ProviderHealthSnapshot,+ error_status_and_message,+};use ai_memory_store::{ApproveAutoImproveProposalResult, AuditLogFilter, AutoImproveProposalOperation,AutoImproveProposalStatus, DecayParams, NewAutoImproveProposal, PagesMode, ReaderPool,@@ -678,6 +681,7 @@ pub fn admin_router_with_sweep_tuning(post(handle_pending_write_reject),).route("/admin/status", get(handle_status))+ .route("/admin/llm-test", post(handle_llm_test)).route("/admin/metrics", get(handle_metrics)).route("/admin/projects", get(handle_list_projects)).route("/admin/project-coordinate", get(handle_project_coordinate))@@ -1308,6 +1312,61 @@ async fn handle_list_projects(State(state): State<Arc<AdminState>>) -> impl Into}}+/// `POST /admin/llm-test` — probe the server's own configured chat provider.+///+/// `ai-memory llm-test` builds a provider from the *caller's* environment,+/// which is the wrong one when the server runs in a container or on another+/// host: its keys exist only there, so following the health report's retry+/// hint failed while the server was healthy (#1639). This runs the provider+/// the server configured, with a fixed short prompt, and answers with the+/// same status class and bounded message the health snapshot records — never+/// the request, the reply text or the provider's full body.+async fn handle_llm_test(State(state): State<Arc<AdminState>>) -> Response {+ let Some(llm) = state.llm.clone() else {+ return (+ StatusCode::SERVICE_UNAVAILABLE,+ Json(serde_json::json!({+ "error": "this server has no LLM provider configured; set one in its config and restart it",+ })),+ )+ .into_response();+ };+ let mut request = ChatRequest::user_prompt("ping".to_owned());+ request.temperature = Some(0.2);+ let started = std::time::Instant::now();+ let elapsed = |started: std::time::Instant| started.elapsed().as_millis() as u64;+ match llm.complete(request).await {+ Ok(response) => Json(serde_json::json!({+ "ok": true,+ "provider": llm.name(),+ "model": response.model,+ "latency_ms": elapsed(started),+ "reply_chars": response.text.chars().count(),+ }))+ .into_response(),+ Err(error) => {+ let (status, message) = error_status_and_message(&error);+ // A provider body can quote the key it rejected. The health+ // snapshot's own copy is bounded but not scrubbed; this one is+ // printed into an operator's terminal, so it goes through the+ // privacy strip first.+ let message = state.wiki.sanitizer().scrub(&message);+ (+ StatusCode::BAD_GATEWAY,+ Json(serde_json::json!({+ "ok": false,+ "provider": llm.name(),+ "model": llm.model(),+ "latency_ms": elapsed(started),+ "error": message,+ "status": status,+ })),+ )+ .into_response()+ }+ }+}+async fn handle_status(State(state): State<Arc<AdminState>>) -> impl IntoResponse {// Count "missing" against the configured embedder's stored triple, so// a model swap cannot read as a fully covered index; with no embedder@@ -14458,6 +14517,7 @@ mod tests {serde_json::json!({"workspace": "default", "project": "scratch"}),),("GET", "/admin/status", serde_json::Value::Null),+ ("POST", "/admin/llm-test", serde_json::Value::Null),("GET", "/admin/metrics", serde_json::Value::Null),("GET",diff --git a/crates/ai-memory-mcp/tests/suite/admin_llm_test.rs b/crates/ai-memory-mcp/tests/suite/admin_llm_test.rsnew file mode 100644index 00000000..af555cd9--- /dev/null+++ b/crates/ai-memory-mcp/tests/suite/admin_llm_test.rs@@ -0,0 +1,182 @@+//! `POST /admin/llm-test` (#1639).+//!+//! The health report's retry hint used to name a client-side `llm-test`, which+//! builds its provider from the caller's environment: with the server in a+//! container or on another host, the keys exist only there, so the advice+//! failed while the server was healthy. The probe now runs on the server, and+//! these pin what it reports — including that a failure never echoes the+//! provider's body or the reply.++use std::sync::Arc;++use ai_memory_llm::{ChatRequest, ChatResponse, LlmError, LlmProvider, LlmResult};+use ai_memory_mcp::{AdminState, admin_router};+use ai_memory_store::{DecayParams, Store};+use ai_memory_wiki::Wiki;+use axum::body::Body;+use axum::http::{Request, StatusCode};+use serde_json::{Value, json};+use tempfile::TempDir;+use tower::ServiceExt;++/// A provider that answers a fixed reply, or fails the way a missing key does.+struct ProbeProvider {+ reply: Result<&'static str, ()>,+}++#[async_trait::async_trait]+impl LlmProvider for ProbeProvider {+ fn name(&self) -> &'static str {+ "openai-compat"+ }++ fn model(&self) -> &str {+ "gpt-test"+ }++ async fn complete(&self, _request: ChatRequest) -> LlmResult<ChatResponse> {+ match self.reply {+ Ok(text) => Ok(ChatResponse {+ text: text.to_owned(),+ usage: None,+ model: "gpt-test".to_owned(),+ }),+ Err(()) => Err(LlmError::Provider {+ status: 401,+ // Composed at runtime: the shape has to look like a key for the+ // scrub to catch it, and a key-shaped literal in the source is+ // a gitleaks finding even when it is fake.+ body: format!("Incorrect API key provided: {}", fake_key()),+ retry_after: None,+ }),+ }+ }++ async fn complete_structured_raw(+ &self,+ _request: ChatRequest,+ _schema: serde_json::Value,+ ) -> LlmResult<serde_json::Value> {+ self.complete(ChatRequest::user_prompt(String::new()))+ .await+ .map(|response| json!({ "text": response.text }))+ }+}++/// A key-shaped value assembled from parts, so no scanner reads it as one.+fn fake_key() -> String {+ format!("sk-{}-{}", "live", "0123456789abcdef")+}++async fn make_admin_state(tmp: &TempDir, llm: Option<Arc<dyn LlmProvider>>) -> AdminState {+ let store = Store::open(tmp.path()).unwrap();+ let wiki = Wiki::new(tmp.path(), store.writer.clone()).unwrap();+ let db_path = store.db_path().to_path_buf();+ AdminState {+ ingest_metrics: Arc::new(ai_memory_core::IngestMetrics::default()),+ mcp_metrics: Arc::new(ai_memory_mcp::McpMetrics::default()),+ writer: store.writer.clone(),+ reader: store.reader.clone(),+ wiki,+ llm,+ auto_improve_require_approval: false,+ auto_improve_review_config: Default::default(),+ embedder: None,+ provider_health: ai_memory_llm::ProviderHealth::default(),+ decay_params: DecayParams::default(),+ contradiction_band_min: ai_memory_consolidate::DEFAULT_CONTRADICTION_SIM_LOW,+ contradiction_band_max: ai_memory_consolidate::DEFAULT_CONTRADICTION_SIM_HIGH,+ data_dir: tmp.path().to_path_buf(),+ db_path,+ bind: "127.0.0.1:0".to_string(),+ home_dir: None,+ bootstrap_lock: Arc::new(tokio::sync::Mutex::new(())),+ token_pepper: None,+ active_project: ai_memory_core::ActiveProject::new(),+ scope_invalidator: None,+ trusted_proxy_identity: false,+ }+}++async fn probe(state: AdminState) -> (StatusCode, Value) {+ let response = admin_router(state)+ .oneshot(+ Request::builder()+ .method("POST")+ .uri("/admin/llm-test")+ .header("content-type", "application/json")+ .body(Body::from("{}"))+ .unwrap(),+ )+ .await+ .unwrap();+ let status = response.status();+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)+ .await+ .unwrap();+ (status, serde_json::from_slice(&bytes).unwrap())+}++#[tokio::test]+async fn a_healthy_server_reports_its_configured_provider() {+ let tmp = TempDir::new().unwrap();+ let state = make_admin_state(&tmp, Some(Arc::new(ProbeProvider { reply: Ok("pong") }))).await;++ let (status, body) = probe(state).await;++ assert_eq!(status, StatusCode::OK, "{body}");+ assert_eq!(body["ok"], json!(true));+ assert_eq!(body["provider"], json!("openai-compat"));+ assert_eq!(body["model"], json!("gpt-test"));+ assert_eq!(+ body["reply_chars"],+ json!(4),+ "the reply is measured, never echoed"+ );+ assert!(body.get("reply").is_none(), "{body}");+ assert!(body["latency_ms"].is_u64(), "{body}");+}++/// The failure class and a bounded message, like the health report: an+/// operator probing from another host must not have the provider's body (which+/// can quote the key) printed into their terminal or their logs.+#[tokio::test]+async fn a_failing_probe_reports_the_class_without_the_provider_body() {+ let tmp = TempDir::new().unwrap();+ let state = make_admin_state(&tmp, Some(Arc::new(ProbeProvider { reply: Err(()) }))).await;++ let (status, body) = probe(state).await;++ assert_eq!(status, StatusCode::BAD_GATEWAY, "{body}");+ assert_eq!(body["ok"], json!(false));+ assert_eq!(body["status"], json!(401));+ assert_eq!(body["provider"], json!("openai-compat"));+ let message = body["error"].as_str().unwrap_or_default();+ assert!(+ message.contains("Incorrect API key provided"),+ "the class is useful: {message}"+ );+ assert!(+ !message.contains(&fake_key()),+ "the key must not reach the operator: {message}"+ );+}++/// A server with no provider configured says so, instead of reporting a probe+/// failure the operator would read as a broken provider.+#[tokio::test]+async fn a_server_without_a_provider_answers_service_unavailable() {+ let tmp = TempDir::new().unwrap();+ let state = make_admin_state(&tmp, None).await;++ let (status, body) = probe(state).await;++ assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "{body}");+ assert!(+ body["error"]+ .as_str()+ .unwrap_or_default()+ .contains("no LLM provider configured"),+ "{body}"+ );+}diff --git a/crates/ai-memory-mcp/tests/suite/mod.rs b/crates/ai-memory-mcp/tests/suite/mod.rsindex 0f79fa33..bf4435fe 100644--- a/crates/ai-memory-mcp/tests/suite/mod.rs+++ b/crates/ai-memory-mcp/tests/suite/mod.rs@@ -8,6 +8,7 @@ mod access_reinforcement;mod admin_audit_log;mod admin_backup;mod admin_bootstrap;+mod admin_llm_test;mod admin_metrics;mod admin_move;mod admin_move_session;diff --git a/docs/llm-providers.md b/docs/llm-providers.mdindex cf22e262..64d4db0b 100644--- a/docs/llm-providers.md+++ b/docs/llm-providers.md@@ -275,6 +275,31 @@ alone does not redirect embeddings; set `AI_MEMORY_EMBEDDING_BASE_URL` too.> startup for that provider; the dynamic value must not be shadowed or> duplicated.+### Probing the server's own provider++`ai-memory llm-test` builds a provider from the shell it runs in. That is the+right check before a key goes into the config, and it works with the server+down — but it is the wrong check when the server runs in a container or on+another host, because the keys exist there and not in your shell. A failure+like `building LLM provider` (no key) or a provider `400 Please pass a valid+API key` says nothing about the server in that case.++`ai-memory llm-test --server` asks the server to run the probe itself, with its+own configured provider, fallback chain and keys, and prints the provider,+model, latency and the error class:++```bash+ai-memory llm-test --server+```++It is the same probe `status` and the health panel name when a provider call+fails, it runs one fixed short prompt, and it takes no `--provider`, `--model`,+`--prompt`, `--base-url` or `--api-key`: a client cannot hand the server a key+nor point it at another URL. On a multi-user server the route is root-only, and+a provider's error body is scrubbed before it is printed, so a key the provider+quoted back never reaches your terminal. Without a configured provider the+server answers `503` and says so.+### Security and gotchas- **Bind address vs. port publish.** When running inside a container, the