Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
fix(llm): truncate embedding input and harden OpenAI-compatible calls
pedrofjrabba229692c2
diff --git a/.env.local.example b/.env.local.examplenew file mode 100644index 00000000..d230315e--- /dev/null+++ b/.env.local.example@@ -0,0 +1,56 @@+# Copy to .env.local at the repo root (gitignored).+# scripts/windows/serve.ps1 loads this file automatically when present.++# Data directory (optional; default: %LOCALAPPDATA%\ai-memory on Windows)+# AI_MEMORY_DATA_DIR=C:\Users\you\.ai-memory++# --- Zero-LLM mode: leave everything below commented out ---++# LLM provider: anthropic | openai | openai-compat | gemini | cursor (fork)+# AI_MEMORY_LLM_PROVIDER=anthropic++# Cursor SDK (local Composer — fork-only; same pool as the IDE)+# Requires: cd scripts/cursor-bridge && npm install+# AI_MEMORY_LLM_PROVIDER=cursor+# CURSOR_API_KEY=+# CURSOR_MODEL=composer-2.5+# CURSOR_MODEL_FAST=false+# CURSOR_AGENT_CWD=C:\path\to\project+# CURSOR_TIMEOUT_MS=120000++# LLM (session consolidation, bootstrap)+# AI_MEMORY_LLM_PROVIDER=anthropic+# AI_MEMORY_LLM_MODEL=claude-haiku-4-5+# ANTHROPIC_API_KEY=sk-ant-...++# OpenAI-compatible (OpenRouter, Ollama, vLLM, …)+# AI_MEMORY_LLM_PROVIDER=openai-compat+# AI_MEMORY_LLM_BASE_URL=https://openrouter.ai/api/v1+# AI_MEMORY_LLM_MODEL=anthropic/claude-haiku-4.5+# LLM_API_KEY=sk-or-v1-...++# Embeddings (hybrid search; optional)+# OpenAI-compatible gateways (e.g. OpenRouter) reuse the same HTTP shape as openai-compat.+# API key: OPENAI_API_KEY or LLM_API_KEY (handy when LLM already uses OpenRouter).+# AI_MEMORY_EMBEDDING_PROVIDER=openai+# AI_MEMORY_EMBEDDING_MODEL=openai/text-embedding-3-small+# AI_MEMORY_EMBEDDING_BASE_URL=https://openrouter.ai/api/v1+# LLM_API_KEY=sk-or-v1-...+# OPENAI_API_KEY=sk-... # alternative if LLM does not use OpenRouter+# AI_MEMORY_EMBEDDING_DIM=1536++# Embeddings via OpenAI directly (no aggregator)+# AI_MEMORY_EMBEDDING_PROVIDER=openai+# AI_MEMORY_EMBEDDING_MODEL=text-embedding-3-small+# AI_MEMORY_EMBEDDING_BASE_URL=https://api.openai.com/v1+# OPENAI_API_KEY=sk-...++# Google Gemini embeddings (https://ai.google.dev/gemini-api/docs/embeddings)+# AI_MEMORY_EMBEDDING_PROVIDER=google+# AI_MEMORY_EMBEDDING_MODEL=gemini-embedding-001+# AI_MEMORY_EMBEDDING_DIM=768+# GEMINI_API_KEY=+# GOOGLE_API_KEY= # accepted alias++# Auth (only when exposing beyond loopback)+# AI_MEMORY_AUTH_TOKEN=diff --git a/crates/ai-memory-cli/src/config.rs b/crates/ai-memory-cli/src/config.rsindex bf15931c..6c21cf93 100644--- a/crates/ai-memory-cli/src/config.rs+++ b/crates/ai-memory-cli/src/config.rs@@ -342,6 +342,23 @@ impl Config {}))}+ /// OpenAI-compatible embedding key: `OPENAI_API_KEY`, then `LLM_API_KEY`+ /// (typical for OpenRouter when `AI_MEMORY_EMBEDDING_BASE_URL` points there).+ fn openai_embedding_api_key(runtime: &RuntimeEnv) -> LlmResult<SecretString> {+ if let Some(key) = runtime.openai_api_key.clone() {+ return Ok(key);+ }+ runtime+ .llm_api_key+ .clone()+ .ok_or_else(|| {+ LlmError::NotConfigured(+ "OPENAI_API_KEY or LLM_API_KEY (OpenRouter) required for openai embeddings"+ .into(),+ )+ })+ }+/// Build the configured embedder settings, if hybrid search is enabled.////// # Errors@@ -373,11 +390,7 @@ impl Config {.embedding_dim.unwrap_or_else(|| ai_memory_llm::default_embedding_dim(provider, &model));let api_key = match provider {- EmbedderChoice::OpenAi => self- .runtime_env- .openai_api_key- .clone()- .ok_or_else(|| LlmError::NotConfigured("OPENAI_API_KEY".into()))?,+ EmbedderChoice::OpenAi => Self::openai_embedding_api_key(&self.runtime_env)?,EmbedderChoice::Voyage => self.runtime_env.voyage_api_key@@ -462,6 +475,7 @@ fn canonicalise_or_keep(p: &Path) -> PathBuf {#[cfg(test)]mod tests {use super::*;+ use secrecy::ExposeSecret;use tempfile::TempDir;#[test]@@ -542,4 +556,27 @@ mod tests {EmbedderChoice::Google);}++ #[test]+ fn openai_embedding_falls_back_to_llm_api_key_for_openrouter() {+ let cfg = Config {+ embedding_provider: Some("openai".into()),+ embedding_model: Some("text-embedding-3-small".into()),+ embedding_base_url: Some("https://openrouter.ai/api/v1".into()),+ runtime_env: RuntimeEnv {+ llm_api_key: Some(SecretString::from("sk-or-test-key")),+ ..RuntimeEnv::default()+ },+ ..Config::default()+ };++ let embedder = cfg.embedder_config().unwrap().unwrap();+ assert_eq!(embedder.provider, EmbedderChoice::OpenAi);+ assert_eq!(embedder.model, "text-embedding-3-small");+ assert_eq!(embedder.api_key.expose_secret(), "sk-or-test-key");+ assert_eq!(+ embedder.base_url.as_deref(),+ Some("https://openrouter.ai/api/v1")+ );+ }}diff --git a/crates/ai-memory-llm/src/embedding.rs b/crates/ai-memory-llm/src/embedding.rsindex c8673736..84097956 100644--- a/crates/ai-memory-llm/src/embedding.rs+++ b/crates/ai-memory-llm/src/embedding.rs@@ -21,7 +21,11 @@ use tracing::debug;use crate::error::{LlmError, LlmResult};use crate::openai::normalize_openai_base;-use crate::text::truncate_with_ellipsis;+use crate::text::{truncate_for_embedding, truncate_with_ellipsis};++/// Conservative per-request input cap for OpenAI-compatible embedding APIs+/// (8192 token server limit; we stay well below with head truncation).+const OPENAI_EMBED_MAX_TOKENS: usize = 5000;/// Provider-agnostic embedding API.///@@ -111,6 +115,75 @@ struct OpenAiEmbeddingDatum {embedding: Vec<f32>,}+/// Parse OpenAI-compatible embedding responses, including OpenRouter error bodies.+fn parse_openai_embedding_values(body: &str, status: u16) -> LlmResult<Vec<f32>> {+ let v: serde_json::Value = serde_json::from_str(body).map_err(|e| {+ LlmError::Provider {+ status,+ body: truncate_with_ellipsis(+ &format!("openai embeddings json: {e}; body={body}"),+ 1024,+ ),+ }+ })?;+ if let Some(err) = v.get("error") {+ let msg = err+ .get("message")+ .and_then(|m| m.as_str())+ .or_else(|| err.as_str())+ .unwrap_or("unknown");+ return Err(LlmError::Provider {+ status,+ body: truncate_with_ellipsis(msg, 1024),+ });+ }+ if let Some(data) = v.get("data").and_then(|d| d.as_array()) {+ let first = data.first().ok_or_else(|| LlmError::Provider {+ status,+ body: truncate_with_ellipsis(+ &format!("openai embeddings data[] empty; body={body}"),+ 512,+ ),+ })?;+ let emb = first+ .get("embedding")+ .and_then(|e| e.as_array())+ .ok_or_else(|| LlmError::Provider {+ status,+ body: truncate_with_ellipsis(+ &format!("missing embedding array in data[0]; body={body}"),+ 512,+ ),+ })?;+ let mut out = Vec::new();+ for n in emb {+ if let Some(f) = n.as_f64() {+ out.push(f as f32);+ }+ }+ if !out.is_empty() {+ return Ok(out);+ }+ }+ // Strict OpenAI shape fallback.+ let parsed: OpenAiEmbeddingResponse = serde_json::from_value(v).map_err(|e| {+ LlmError::Provider {+ status,+ body: truncate_with_ellipsis(+ &format!("openai embeddings shape: {e}; body={body}"),+ 1024,+ ),+ }+ })?;+ let first = parsed.data.into_iter().next().ok_or_else(|| {+ LlmError::UnexpectedShape(format!(+ "openai response had no data[0]; body={}",+ truncate_with_ellipsis(body, 512)+ ))+ })?;+ Ok(first.embedding)+}+#[async_trait]impl Embedder for OpenAiEmbedder {fn provider(&self) -> &'static str {@@ -126,40 +199,61 @@ impl Embedder for OpenAiEmbedder {}async fn embed(&self, text: &str) -> LlmResult<Vec<f32>> {+ let input = truncate_for_embedding(text, OPENAI_EMBED_MAX_TOKENS);let url = normalize_openai_base(&self.base_url, "embeddings");- debug!(url, model = %self.model, "POST openai/embeddings");+ debug!(+ url,+ model = %self.model,+ input_chars = input.len(),+ "POST openai/embeddings"+ );let req = OpenAiEmbeddingRequest {- input: text,+ input: &input,model: &self.model,};- let resp = self- .client- .post(&url)- .bearer_auth(self.api_key.expose_secret())- .json(&req)- .send()- .await?;- let status = resp.status();- if !status.is_success() {+ let mut attempt = 0u32;+ loop {+ let resp = self+ .client+ .post(&url)+ .bearer_auth(self.api_key.expose_secret())+ .json(&req)+ .send()+ .await?;+ let status = resp.status();+ if status.as_u16() == 429 && attempt < 5 {+ attempt += 1;+ let delay = Duration::from_secs(2u64.saturating_pow(attempt));+ debug!(attempt, ?delay, "openai embeddings rate-limited; retrying");+ tokio::time::sleep(delay).await;+ continue;+ }+ // Client errors (e.g. input > 8192 tokens) are not retried.+ if status.as_u16() == 400 {+ let body = resp.text().await.unwrap_or_default();+ return Err(LlmError::Provider {+ status: status.as_u16(),+ body: truncate_with_ellipsis(&body, 1024),+ });+ }+ if !status.is_success() {+ let body = resp.text().await.unwrap_or_default();+ return Err(LlmError::Provider {+ status: status.as_u16(),+ body: truncate_with_ellipsis(&body, 1024),+ });+ }let body = resp.text().await.unwrap_or_default();- return Err(LlmError::Provider {- status: status.as_u16(),- body: truncate_with_ellipsis(&body, 1024),- });- }- let parsed: OpenAiEmbeddingResponse = resp.json().await?;- let first =- parsed.data.into_iter().next().ok_or_else(|| {- LlmError::UnexpectedShape("openai response had no data[0]".into())- })?;- if first.embedding.len() as u32 != self.dim {- return Err(LlmError::UnexpectedShape(format!(- "expected dim {}, got {}",- self.dim,- first.embedding.len()- )));+ let values = parse_openai_embedding_values(&body, status.as_u16())?;+ if values.len() as u32 != self.dim {+ return Err(LlmError::UnexpectedShape(format!(+ "expected dim {}, got {}",+ self.dim,+ values.len()+ )));+ }+ return Ok(normalise(values));}- Ok(normalise(first.embedding))}}@@ -387,4 +481,18 @@ mod tests {let b = e.embed("compile not retrieve").await.unwrap();assert_eq!(a, b);}++ #[test]+ fn parse_openai_embedding_accepts_data_array() {+ let body = r#"{"data":[{"embedding":[0.1,0.2,0.3]}]}"#;+ let v = parse_openai_embedding_values(body, 200).unwrap();+ assert_eq!(v.len(), 3);+ }++ #[test]+ fn parse_openai_embedding_accepts_openrouter_error() {+ let body = r#"{"error":{"message":"Provider returned error","code":502}}"#;+ let err = parse_openai_embedding_values(body, 502).unwrap_err();+ assert!(matches!(err, LlmError::Provider { status: 502, .. }));+ }}diff --git a/crates/ai-memory-llm/src/google.rs b/crates/ai-memory-llm/src/google.rsindex 61473e59..dddc9119 100644--- a/crates/ai-memory-llm/src/google.rs+++ b/crates/ai-memory-llm/src/google.rs@@ -84,31 +84,41 @@ impl GoogleEmbedder {};debug!(url, model = %self.model, ?task_type, "POST google/embedContent");- let resp = self- .client- .post(&url)- .header("x-goog-api-key", self.api_key.expose_secret())- .json(&body)- .send()- .await?;- let status = resp.status();- if !status.is_success() {- let body_text = resp.text().await.unwrap_or_default();- return Err(LlmError::Provider {- status: status.as_u16(),- body: truncate_with_ellipsis(&body_text, 1024),- });- }- let parsed: GeminiEmbedResponse = resp.json().await?;- let values = parsed.embedding.values;- if values.len() as u32 != self.dim {- return Err(LlmError::UnexpectedShape(format!(- "expected dim {}, got {}",- self.dim,- values.len()- )));+ let mut attempt = 0u32;+ loop {+ let resp = self+ .client+ .post(&url)+ .header("x-goog-api-key", self.api_key.expose_secret())+ .json(&body)+ .send()+ .await?;+ let status = resp.status();+ if status.as_u16() == 429 && attempt < 5 {+ attempt += 1;+ let delay = Duration::from_secs(2u64.saturating_pow(attempt));+ debug!(attempt, ?delay, "google embedContent rate-limited; retrying");+ tokio::time::sleep(delay).await;+ continue;+ }+ if !status.is_success() {+ let body_text = resp.text().await.unwrap_or_default();+ return Err(LlmError::Provider {+ status: status.as_u16(),+ body: truncate_with_ellipsis(&body_text, 1024),+ });+ }+ let parsed: GeminiEmbedResponse = resp.json().await?;+ let values = parsed.embedding.values;+ if values.len() as u32 != self.dim {+ return Err(LlmError::UnexpectedShape(format!(+ "expected dim {}, got {}",+ self.dim,+ values.len()+ )));+ }+ return Ok(normalise(values));}- Ok(normalise(values))}}diff --git a/crates/ai-memory-llm/src/text.rs b/crates/ai-memory-llm/src/text.rsindex d3b58cff..b04d60c6 100644--- a/crates/ai-memory-llm/src/text.rs+++ b/crates/ai-memory-llm/src/text.rs@@ -30,6 +30,19 @@ pub(crate) fn suffix_within_bytes(s: &str, max_bytes: usize) -> &str {&s[start..]}+/// Truncate text so it fits provider input limits (e.g. OpenAI/OpenRouter+/// 8192 tokens). Dense markdown/code can approach ~1 byte/token, so we cap+/// both a token budget (optimistic prose) and a hard byte ceiling.+pub(crate) fn truncate_for_embedding(text: &str, max_tokens: usize) -> String {+ const HARD_MAX_BYTES: usize = 8_000;+ let token_budget_bytes = max_tokens.saturating_mul(3);+ let max_bytes = token_budget_bytes.min(HARD_MAX_BYTES);+ if text.len() <= max_bytes {+ return text.to_string();+ }+ truncate_with_ellipsis(text, max_bytes.saturating_sub(1))+}+#[cfg(test)]mod tests {use super::*;@@ -53,4 +66,13 @@ mod tests {let s = format!("é{}", "x".repeat(1023));assert_eq!(suffix_within_bytes(&s, 1024), "x".repeat(1023));}++ #[test]+ fn truncate_for_embedding_caps_long_input() {+ let long = "x".repeat(50_000);+ let out = truncate_for_embedding(&long, 6000);+ assert!(out.ends_with('…'));+ assert!(out.len() < long.len());+ assert!(out.len() <= 8_000 + 4);+ }}diff --git a/docs/install.md b/docs/install.mdindex 5e208c2f..1a37f114 100644--- a/docs/install.md+++ b/docs/install.md@@ -357,6 +357,7 @@ If you set only the provider, ai-memory picks a sensible default:| `AI_MEMORY_LLM_PROVIDER=openai` | `gpt-5.4-mini` | Cheaper + faster alternative. Same parse reliability; mild over-classification on thin sessions. || `AI_MEMORY_LLM_PROVIDER=gemini` | `gemini-2.5-flash` | Google's hosted option with a generous free tier. ai-memory disables Gemini 2.5 Flash's default dynamic thinking so hidden thought tokens do not truncate strict JSON. Set `GEMINI_API_KEY` (or `GOOGLE_API_KEY`). || `AI_MEMORY_EMBEDDING_PROVIDER=openai` | `text-embedding-3-small` (1536-dim) | 5× cheaper than `-3-large` with marginal recall loss. |+| `AI_MEMORY_EMBEDDING_PROVIDER=openai` + `AI_MEMORY_EMBEDDING_BASE_URL=https://openrouter.ai/api/v1` | `openai/text-embedding-3-small` via [OpenRouter](https://openrouter.ai) | Same API key as `LLM_API_KEY` / `OPENAI_API_KEY`; no código extra além do cliente OpenAI-compat. || `AI_MEMORY_EMBEDDING_PROVIDER=voyage` | `voyage-3` (1024-dim) | Voyage's current general-purpose recommendation. || `AI_MEMORY_EMBEDDING_PROVIDER=google` / `gemini` | `gemini-embedding-001` (768-dim) | Google-hosted embeddings via `embedContent`. Set `GEMINI_API_KEY` (or `GOOGLE_API_KEY`). |