Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(llm): truncate embedding input and harden OpenAI-compatible calls

pedrofjrabba229692c2
diff --git a/.env.local.example b/.env.local.example
new file mode 100644
index 00000000..d230315e
--- /dev/null
+++ b/.env.local.example
@@ -0,0 +1,56 @@
+# Copy to .env.local at the repo root (gitignored).
+# scripts/windows/serve.ps1 loads this file automatically when present.
+
+# Data directory (optional; default: %LOCALAPPDATA%\ai-memory on Windows)
+# AI_MEMORY_DATA_DIR=C:\Users\you\.ai-memory
+
+# --- Zero-LLM mode: leave everything below commented out ---
+
+# LLM provider: anthropic | openai | openai-compat | gemini | cursor (fork)
+# AI_MEMORY_LLM_PROVIDER=anthropic
+
+# Cursor SDK (local Composer — fork-only; same pool as the IDE)
+# Requires: cd scripts/cursor-bridge && npm install
+# AI_MEMORY_LLM_PROVIDER=cursor
+# CURSOR_API_KEY=
+# CURSOR_MODEL=composer-2.5
+# CURSOR_MODEL_FAST=false
+# CURSOR_AGENT_CWD=C:\path\to\project
+# CURSOR_TIMEOUT_MS=120000
+
+# LLM (session consolidation, bootstrap)
+# AI_MEMORY_LLM_PROVIDER=anthropic
+# AI_MEMORY_LLM_MODEL=claude-haiku-4-5
+# ANTHROPIC_API_KEY=sk-ant-...
+
+# OpenAI-compatible (OpenRouter, Ollama, vLLM, …)
+# AI_MEMORY_LLM_PROVIDER=openai-compat
+# AI_MEMORY_LLM_BASE_URL=https://openrouter.ai/api/v1
+# AI_MEMORY_LLM_MODEL=anthropic/claude-haiku-4.5
+# LLM_API_KEY=sk-or-v1-...
+
+# Embeddings (hybrid search; optional)
+# OpenAI-compatible gateways (e.g. OpenRouter) reuse the same HTTP shape as openai-compat.
+# API key: OPENAI_API_KEY or LLM_API_KEY (handy when LLM already uses OpenRouter).
+# AI_MEMORY_EMBEDDING_PROVIDER=openai
+# AI_MEMORY_EMBEDDING_MODEL=openai/text-embedding-3-small
+# AI_MEMORY_EMBEDDING_BASE_URL=https://openrouter.ai/api/v1
+# LLM_API_KEY=sk-or-v1-...
+# OPENAI_API_KEY=sk-... # alternative if LLM does not use OpenRouter
+# AI_MEMORY_EMBEDDING_DIM=1536
+
+# Embeddings via OpenAI directly (no aggregator)
+# AI_MEMORY_EMBEDDING_PROVIDER=openai
+# AI_MEMORY_EMBEDDING_MODEL=text-embedding-3-small
+# AI_MEMORY_EMBEDDING_BASE_URL=https://api.openai.com/v1
+# OPENAI_API_KEY=sk-...
+
+# Google Gemini embeddings (https://ai.google.dev/gemini-api/docs/embeddings)
+# AI_MEMORY_EMBEDDING_PROVIDER=google
+# AI_MEMORY_EMBEDDING_MODEL=gemini-embedding-001
+# AI_MEMORY_EMBEDDING_DIM=768
+# GEMINI_API_KEY=
+# GOOGLE_API_KEY= # accepted alias
+
+# Auth (only when exposing beyond loopback)
+# AI_MEMORY_AUTH_TOKEN=
diff --git a/crates/ai-memory-cli/src/config.rs b/crates/ai-memory-cli/src/config.rs
index bf15931c..6c21cf93 100644
--- a/crates/ai-memory-cli/src/config.rs
+++ b/crates/ai-memory-cli/src/config.rs
@@ -342,6 +342,23 @@ impl Config {
}))
}
+ /// OpenAI-compatible embedding key: `OPENAI_API_KEY`, then `LLM_API_KEY`
+ /// (typical for OpenRouter when `AI_MEMORY_EMBEDDING_BASE_URL` points there).
+ fn openai_embedding_api_key(runtime: &RuntimeEnv) -> LlmResult<SecretString> {
+ if let Some(key) = runtime.openai_api_key.clone() {
+ return Ok(key);
+ }
+ runtime
+ .llm_api_key
+ .clone()
+ .ok_or_else(|| {
+ LlmError::NotConfigured(
+ "OPENAI_API_KEY or LLM_API_KEY (OpenRouter) required for openai embeddings"
+ .into(),
+ )
+ })
+ }
+
/// Build the configured embedder settings, if hybrid search is enabled.
///
/// # Errors
@@ -373,11 +390,7 @@ impl Config {
.embedding_dim
.unwrap_or_else(|| ai_memory_llm::default_embedding_dim(provider, &model));
let api_key = match provider {
- EmbedderChoice::OpenAi => self
- .runtime_env
- .openai_api_key
- .clone()
- .ok_or_else(|| LlmError::NotConfigured("OPENAI_API_KEY".into()))?,
+ EmbedderChoice::OpenAi => Self::openai_embedding_api_key(&self.runtime_env)?,
EmbedderChoice::Voyage => self
.runtime_env
.voyage_api_key
@@ -462,6 +475,7 @@ fn canonicalise_or_keep(p: &Path) -> PathBuf {
#[cfg(test)]
mod tests {
use super::*;
+ use secrecy::ExposeSecret;
use tempfile::TempDir;
#[test]
@@ -542,4 +556,27 @@ mod tests {
EmbedderChoice::Google
);
}
+
+ #[test]
+ fn openai_embedding_falls_back_to_llm_api_key_for_openrouter() {
+ let cfg = Config {
+ embedding_provider: Some("openai".into()),
+ embedding_model: Some("text-embedding-3-small".into()),
+ embedding_base_url: Some("https://openrouter.ai/api/v1".into()),
+ runtime_env: RuntimeEnv {
+ llm_api_key: Some(SecretString::from("sk-or-test-key")),
+ ..RuntimeEnv::default()
+ },
+ ..Config::default()
+ };
+
+ let embedder = cfg.embedder_config().unwrap().unwrap();
+ assert_eq!(embedder.provider, EmbedderChoice::OpenAi);
+ assert_eq!(embedder.model, "text-embedding-3-small");
+ assert_eq!(embedder.api_key.expose_secret(), "sk-or-test-key");
+ assert_eq!(
+ embedder.base_url.as_deref(),
+ Some("https://openrouter.ai/api/v1")
+ );
+ }
}
diff --git a/crates/ai-memory-llm/src/embedding.rs b/crates/ai-memory-llm/src/embedding.rs
index c8673736..84097956 100644
--- a/crates/ai-memory-llm/src/embedding.rs
+++ b/crates/ai-memory-llm/src/embedding.rs
@@ -21,7 +21,11 @@ use tracing::debug;
use crate::error::{LlmError, LlmResult};
use crate::openai::normalize_openai_base;
-use crate::text::truncate_with_ellipsis;
+use crate::text::{truncate_for_embedding, truncate_with_ellipsis};
+
+/// Conservative per-request input cap for OpenAI-compatible embedding APIs
+/// (8192 token server limit; we stay well below with head truncation).
+const OPENAI_EMBED_MAX_TOKENS: usize = 5000;
/// Provider-agnostic embedding API.
///
@@ -111,6 +115,75 @@ struct OpenAiEmbeddingDatum {
embedding: Vec<f32>,
}
+/// Parse OpenAI-compatible embedding responses, including OpenRouter error bodies.
+fn parse_openai_embedding_values(body: &str, status: u16) -> LlmResult<Vec<f32>> {
+ let v: serde_json::Value = serde_json::from_str(body).map_err(|e| {
+ LlmError::Provider {
+ status,
+ body: truncate_with_ellipsis(
+ &format!("openai embeddings json: {e}; body={body}"),
+ 1024,
+ ),
+ }
+ })?;
+ if let Some(err) = v.get("error") {
+ let msg = err
+ .get("message")
+ .and_then(|m| m.as_str())
+ .or_else(|| err.as_str())
+ .unwrap_or("unknown");
+ return Err(LlmError::Provider {
+ status,
+ body: truncate_with_ellipsis(msg, 1024),
+ });
+ }
+ if let Some(data) = v.get("data").and_then(|d| d.as_array()) {
+ let first = data.first().ok_or_else(|| LlmError::Provider {
+ status,
+ body: truncate_with_ellipsis(
+ &format!("openai embeddings data[] empty; body={body}"),
+ 512,
+ ),
+ })?;
+ let emb = first
+ .get("embedding")
+ .and_then(|e| e.as_array())
+ .ok_or_else(|| LlmError::Provider {
+ status,
+ body: truncate_with_ellipsis(
+ &format!("missing embedding array in data[0]; body={body}"),
+ 512,
+ ),
+ })?;
+ let mut out = Vec::new();
+ for n in emb {
+ if let Some(f) = n.as_f64() {
+ out.push(f as f32);
+ }
+ }
+ if !out.is_empty() {
+ return Ok(out);
+ }
+ }
+ // Strict OpenAI shape fallback.
+ let parsed: OpenAiEmbeddingResponse = serde_json::from_value(v).map_err(|e| {
+ LlmError::Provider {
+ status,
+ body: truncate_with_ellipsis(
+ &format!("openai embeddings shape: {e}; body={body}"),
+ 1024,
+ ),
+ }
+ })?;
+ let first = parsed.data.into_iter().next().ok_or_else(|| {
+ LlmError::UnexpectedShape(format!(
+ "openai response had no data[0]; body={}",
+ truncate_with_ellipsis(body, 512)
+ ))
+ })?;
+ Ok(first.embedding)
+}
+
#[async_trait]
impl Embedder for OpenAiEmbedder {
fn provider(&self) -> &'static str {
@@ -126,40 +199,61 @@ impl Embedder for OpenAiEmbedder {
}
async fn embed(&self, text: &str) -> LlmResult<Vec<f32>> {
+ let input = truncate_for_embedding(text, OPENAI_EMBED_MAX_TOKENS);
let url = normalize_openai_base(&self.base_url, "embeddings");
- debug!(url, model = %self.model, "POST openai/embeddings");
+ debug!(
+ url,
+ model = %self.model,
+ input_chars = input.len(),
+ "POST openai/embeddings"
+ );
let req = OpenAiEmbeddingRequest {
- input: text,
+ input: &input,
model: &self.model,
};
- let resp = self
- .client
- .post(&url)
- .bearer_auth(self.api_key.expose_secret())
- .json(&req)
- .send()
- .await?;
- let status = resp.status();
- if !status.is_success() {
+ let mut attempt = 0u32;
+ loop {
+ let resp = self
+ .client
+ .post(&url)
+ .bearer_auth(self.api_key.expose_secret())
+ .json(&req)
+ .send()
+ .await?;
+ let status = resp.status();
+ if status.as_u16() == 429 && attempt < 5 {
+ attempt += 1;
+ let delay = Duration::from_secs(2u64.saturating_pow(attempt));
+ debug!(attempt, ?delay, "openai embeddings rate-limited; retrying");
+ tokio::time::sleep(delay).await;
+ continue;
+ }
+ // Client errors (e.g. input > 8192 tokens) are not retried.
+ if status.as_u16() == 400 {
+ let body = resp.text().await.unwrap_or_default();
+ return Err(LlmError::Provider {
+ status: status.as_u16(),
+ body: truncate_with_ellipsis(&body, 1024),
+ });
+ }
+ if !status.is_success() {
+ let body = resp.text().await.unwrap_or_default();
+ return Err(LlmError::Provider {
+ status: status.as_u16(),
+ body: truncate_with_ellipsis(&body, 1024),
+ });
+ }
let body = resp.text().await.unwrap_or_default();
- return Err(LlmError::Provider {
- status: status.as_u16(),
- body: truncate_with_ellipsis(&body, 1024),
- });
- }
- let parsed: OpenAiEmbeddingResponse = resp.json().await?;
- let first =
- parsed.data.into_iter().next().ok_or_else(|| {
- LlmError::UnexpectedShape("openai response had no data[0]".into())
- })?;
- if first.embedding.len() as u32 != self.dim {
- return Err(LlmError::UnexpectedShape(format!(
- "expected dim {}, got {}",
- self.dim,
- first.embedding.len()
- )));
+ let values = parse_openai_embedding_values(&body, status.as_u16())?;
+ if values.len() as u32 != self.dim {
+ return Err(LlmError::UnexpectedShape(format!(
+ "expected dim {}, got {}",
+ self.dim,
+ values.len()
+ )));
+ }
+ return Ok(normalise(values));
}
- Ok(normalise(first.embedding))
}
}
@@ -387,4 +481,18 @@ mod tests {
let b = e.embed("compile not retrieve").await.unwrap();
assert_eq!(a, b);
}
+
+ #[test]
+ fn parse_openai_embedding_accepts_data_array() {
+ let body = r#"{"data":[{"embedding":[0.1,0.2,0.3]}]}"#;
+ let v = parse_openai_embedding_values(body, 200).unwrap();
+ assert_eq!(v.len(), 3);
+ }
+
+ #[test]
+ fn parse_openai_embedding_accepts_openrouter_error() {
+ let body = r#"{"error":{"message":"Provider returned error","code":502}}"#;
+ let err = parse_openai_embedding_values(body, 502).unwrap_err();
+ assert!(matches!(err, LlmError::Provider { status: 502, .. }));
+ }
}
diff --git a/crates/ai-memory-llm/src/google.rs b/crates/ai-memory-llm/src/google.rs
index 61473e59..dddc9119 100644
--- a/crates/ai-memory-llm/src/google.rs
+++ b/crates/ai-memory-llm/src/google.rs
@@ -84,31 +84,41 @@ impl GoogleEmbedder {
};
debug!(url, model = %self.model, ?task_type, "POST google/embedContent");
- let resp = self
- .client
- .post(&url)
- .header("x-goog-api-key", self.api_key.expose_secret())
- .json(&body)
- .send()
- .await?;
- let status = resp.status();
- if !status.is_success() {
- let body_text = resp.text().await.unwrap_or_default();
- return Err(LlmError::Provider {
- status: status.as_u16(),
- body: truncate_with_ellipsis(&body_text, 1024),
- });
- }
- let parsed: GeminiEmbedResponse = resp.json().await?;
- let values = parsed.embedding.values;
- if values.len() as u32 != self.dim {
- return Err(LlmError::UnexpectedShape(format!(
- "expected dim {}, got {}",
- self.dim,
- values.len()
- )));
+ let mut attempt = 0u32;
+ loop {
+ let resp = self
+ .client
+ .post(&url)
+ .header("x-goog-api-key", self.api_key.expose_secret())
+ .json(&body)
+ .send()
+ .await?;
+ let status = resp.status();
+ if status.as_u16() == 429 && attempt < 5 {
+ attempt += 1;
+ let delay = Duration::from_secs(2u64.saturating_pow(attempt));
+ debug!(attempt, ?delay, "google embedContent rate-limited; retrying");
+ tokio::time::sleep(delay).await;
+ continue;
+ }
+ if !status.is_success() {
+ let body_text = resp.text().await.unwrap_or_default();
+ return Err(LlmError::Provider {
+ status: status.as_u16(),
+ body: truncate_with_ellipsis(&body_text, 1024),
+ });
+ }
+ let parsed: GeminiEmbedResponse = resp.json().await?;
+ let values = parsed.embedding.values;
+ if values.len() as u32 != self.dim {
+ return Err(LlmError::UnexpectedShape(format!(
+ "expected dim {}, got {}",
+ self.dim,
+ values.len()
+ )));
+ }
+ return Ok(normalise(values));
}
- Ok(normalise(values))
}
}
diff --git a/crates/ai-memory-llm/src/text.rs b/crates/ai-memory-llm/src/text.rs
index d3b58cff..b04d60c6 100644
--- a/crates/ai-memory-llm/src/text.rs
+++ b/crates/ai-memory-llm/src/text.rs
@@ -30,6 +30,19 @@ pub(crate) fn suffix_within_bytes(s: &str, max_bytes: usize) -> &str {
&s[start..]
}
+/// Truncate text so it fits provider input limits (e.g. OpenAI/OpenRouter
+/// 8192 tokens). Dense markdown/code can approach ~1 byte/token, so we cap
+/// both a token budget (optimistic prose) and a hard byte ceiling.
+pub(crate) fn truncate_for_embedding(text: &str, max_tokens: usize) -> String {
+ const HARD_MAX_BYTES: usize = 8_000;
+ let token_budget_bytes = max_tokens.saturating_mul(3);
+ let max_bytes = token_budget_bytes.min(HARD_MAX_BYTES);
+ if text.len() <= max_bytes {
+ return text.to_string();
+ }
+ truncate_with_ellipsis(text, max_bytes.saturating_sub(1))
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -53,4 +66,13 @@ mod tests {
let s = format!("é{}", "x".repeat(1023));
assert_eq!(suffix_within_bytes(&s, 1024), "x".repeat(1023));
}
+
+ #[test]
+ fn truncate_for_embedding_caps_long_input() {
+ let long = "x".repeat(50_000);
+ let out = truncate_for_embedding(&long, 6000);
+ assert!(out.ends_with('…'));
+ assert!(out.len() < long.len());
+ assert!(out.len() <= 8_000 + 4);
+ }
}
diff --git a/docs/install.md b/docs/install.md
index 5e208c2f..1a37f114 100644
--- a/docs/install.md
+++ b/docs/install.md
@@ -357,6 +357,7 @@ If you set only the provider, ai-memory picks a sensible default:
| `AI_MEMORY_LLM_PROVIDER=openai` | `gpt-5.4-mini` | Cheaper + faster alternative. Same parse reliability; mild over-classification on thin sessions. |
| `AI_MEMORY_LLM_PROVIDER=gemini` | `gemini-2.5-flash` | Google's hosted option with a generous free tier. ai-memory disables Gemini 2.5 Flash's default dynamic thinking so hidden thought tokens do not truncate strict JSON. Set `GEMINI_API_KEY` (or `GOOGLE_API_KEY`). |
| `AI_MEMORY_EMBEDDING_PROVIDER=openai` | `text-embedding-3-small` (1536-dim) | 5× cheaper than `-3-large` with marginal recall loss. |
+| `AI_MEMORY_EMBEDDING_PROVIDER=openai` + `AI_MEMORY_EMBEDDING_BASE_URL=https://openrouter.ai/api/v1` | `openai/text-embedding-3-small` via [OpenRouter](https://openrouter.ai) | Same API key as `LLM_API_KEY` / `OPENAI_API_KEY`; no código extra além do cliente OpenAI-compat. |
| `AI_MEMORY_EMBEDDING_PROVIDER=voyage` | `voyage-3` (1024-dim) | Voyage's current general-purpose recommendation. |
| `AI_MEMORY_EMBEDDING_PROVIDER=google` / `gemini` | `gemini-embedding-001` (768-dim) | Google-hosted embeddings via `embedContent`. Set `GEMINI_API_KEY` (or `GOOGLE_API_KEY`). |