Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(embed): harden openai-compatible embedding calls

AkitaOnRails265a7a3b96d1
diff --git a/.env.local.example b/.env.local.example
deleted file mode 100644
index d230315e..00000000
--- a/.env.local.example
+++ /dev/null
@@ -1,56 +0,0 @@
-# Copy to .env.local at the repo root (gitignored).
-# scripts/windows/serve.ps1 loads this file automatically when present.
-
-# Data directory (optional; default: %LOCALAPPDATA%\ai-memory on Windows)
-# AI_MEMORY_DATA_DIR=C:\Users\you\.ai-memory
-
-# --- Zero-LLM mode: leave everything below commented out ---
-
-# LLM provider: anthropic | openai | openai-compat | gemini | cursor (fork)
-# AI_MEMORY_LLM_PROVIDER=anthropic
-
-# Cursor SDK (local Composer — fork-only; same pool as the IDE)
-# Requires: cd scripts/cursor-bridge && npm install
-# AI_MEMORY_LLM_PROVIDER=cursor
-# CURSOR_API_KEY=
-# CURSOR_MODEL=composer-2.5
-# CURSOR_MODEL_FAST=false
-# CURSOR_AGENT_CWD=C:\path\to\project
-# CURSOR_TIMEOUT_MS=120000
-
-# LLM (session consolidation, bootstrap)
-# AI_MEMORY_LLM_PROVIDER=anthropic
-# AI_MEMORY_LLM_MODEL=claude-haiku-4-5
-# ANTHROPIC_API_KEY=sk-ant-...
-
-# OpenAI-compatible (OpenRouter, Ollama, vLLM, …)
-# AI_MEMORY_LLM_PROVIDER=openai-compat
-# AI_MEMORY_LLM_BASE_URL=https://openrouter.ai/api/v1
-# AI_MEMORY_LLM_MODEL=anthropic/claude-haiku-4.5
-# LLM_API_KEY=sk-or-v1-...
-
-# Embeddings (hybrid search; optional)
-# OpenAI-compatible gateways (e.g. OpenRouter) reuse the same HTTP shape as openai-compat.
-# API key: OPENAI_API_KEY or LLM_API_KEY (handy when LLM already uses OpenRouter).
-# AI_MEMORY_EMBEDDING_PROVIDER=openai
-# AI_MEMORY_EMBEDDING_MODEL=openai/text-embedding-3-small
-# AI_MEMORY_EMBEDDING_BASE_URL=https://openrouter.ai/api/v1
-# LLM_API_KEY=sk-or-v1-...
-# OPENAI_API_KEY=sk-... # alternative if LLM does not use OpenRouter
-# AI_MEMORY_EMBEDDING_DIM=1536
-
-# Embeddings via OpenAI directly (no aggregator)
-# AI_MEMORY_EMBEDDING_PROVIDER=openai
-# AI_MEMORY_EMBEDDING_MODEL=text-embedding-3-small
-# AI_MEMORY_EMBEDDING_BASE_URL=https://api.openai.com/v1
-# OPENAI_API_KEY=sk-...
-
-# Google Gemini embeddings (https://ai.google.dev/gemini-api/docs/embeddings)
-# AI_MEMORY_EMBEDDING_PROVIDER=google
-# AI_MEMORY_EMBEDDING_MODEL=gemini-embedding-001
-# AI_MEMORY_EMBEDDING_DIM=768
-# GEMINI_API_KEY=
-# GOOGLE_API_KEY= # accepted alias
-
-# Auth (only when exposing beyond loopback)
-# AI_MEMORY_AUTH_TOKEN=
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8e8584fa..366c9bdc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
- Filesystem watcher now reindexes a project when backends report only a
parent-directory event, improving external editor capture on macOS/FSEvents.
+- OpenAI-compatible embedding calls now truncate oversized page bodies, surface
+ provider errors returned in HTTP 200 bodies, retry bounded HTTP 429 responses,
+ and may reuse `LLM_API_KEY` when a custom embedding base URL is configured.
## [0.2.0] - 2026-05-26
### Added
diff --git a/crates/ai-memory-cli/src/config.rs b/crates/ai-memory-cli/src/config.rs
index 6c21cf93..7f9d5639 100644
--- a/crates/ai-memory-cli/src/config.rs
+++ b/crates/ai-memory-cli/src/config.rs
@@ -342,21 +342,22 @@ impl Config {
}))
}
- /// OpenAI-compatible embedding key: `OPENAI_API_KEY`, then `LLM_API_KEY`
- /// (typical for OpenRouter when `AI_MEMORY_EMBEDDING_BASE_URL` points there).
- fn openai_embedding_api_key(runtime: &RuntimeEnv) -> LlmResult<SecretString> {
- if let Some(key) = runtime.openai_api_key.clone() {
+ /// OpenAI-compatible embedding key. Direct OpenAI keeps requiring
+ /// `OPENAI_API_KEY`; a custom embedding base URL may reuse `LLM_API_KEY`
+ /// for gateways such as OpenRouter.
+ fn openai_embedding_api_key(&self) -> LlmResult<SecretString> {
+ if let Some(key) = self.runtime_env.openai_api_key.clone() {
return Ok(key);
}
- runtime
- .llm_api_key
- .clone()
- .ok_or_else(|| {
- LlmError::NotConfigured(
- "OPENAI_API_KEY or LLM_API_KEY (OpenRouter) required for openai embeddings"
- .into(),
- )
- })
+ if non_empty(self.embedding_base_url.as_deref()).is_some() {
+ if let Some(key) = self.runtime_env.llm_api_key.clone() {
+ return Ok(key);
+ }
+ return Err(LlmError::NotConfigured(
+ "OPENAI_API_KEY or LLM_API_KEY required for openai-compatible embeddings".into(),
+ ));
+ }
+ Err(LlmError::NotConfigured("OPENAI_API_KEY".into()))
}
/// Build the configured embedder settings, if hybrid search is enabled.
@@ -390,7 +391,7 @@ impl Config {
.embedding_dim
.unwrap_or_else(|| ai_memory_llm::default_embedding_dim(provider, &model));
let api_key = match provider {
- EmbedderChoice::OpenAi => Self::openai_embedding_api_key(&self.runtime_env)?,
+ EmbedderChoice::OpenAi => self.openai_embedding_api_key()?,
EmbedderChoice::Voyage => self
.runtime_env
.voyage_api_key
@@ -579,4 +580,19 @@ mod tests {
Some("https://openrouter.ai/api/v1")
);
}
+
+ #[test]
+ fn openai_embedding_does_not_use_llm_api_key_without_custom_base_url() {
+ let cfg = Config {
+ embedding_provider: Some("openai".into()),
+ runtime_env: RuntimeEnv {
+ llm_api_key: Some(SecretString::from("sk-or-test-key")),
+ ..RuntimeEnv::default()
+ },
+ ..Config::default()
+ };
+
+ let err = cfg.embedder_config().unwrap_err();
+ assert!(matches!(err, LlmError::NotConfigured(msg) if msg == "OPENAI_API_KEY"));
+ }
}
diff --git a/crates/ai-memory-llm/src/embedding.rs b/crates/ai-memory-llm/src/embedding.rs
index 84097956..1714b233 100644
--- a/crates/ai-memory-llm/src/embedding.rs
+++ b/crates/ai-memory-llm/src/embedding.rs
@@ -117,14 +117,9 @@ struct OpenAiEmbeddingDatum {
/// Parse OpenAI-compatible embedding responses, including OpenRouter error bodies.
fn parse_openai_embedding_values(body: &str, status: u16) -> LlmResult<Vec<f32>> {
- let v: serde_json::Value = serde_json::from_str(body).map_err(|e| {
- LlmError::Provider {
- status,
- body: truncate_with_ellipsis(
- &format!("openai embeddings json: {e}; body={body}"),
- 1024,
- ),
- }
+ let v: serde_json::Value = serde_json::from_str(body).map_err(|e| LlmError::Provider {
+ status,
+ body: truncate_with_ellipsis(&format!("openai embeddings json: {e}; body={body}"), 1024),
})?;
if let Some(err) = v.get("error") {
let msg = err
@@ -166,15 +161,14 @@ fn parse_openai_embedding_values(body: &str, status: u16) -> LlmResult<Vec<f32>>
}
}
// Strict OpenAI shape fallback.
- let parsed: OpenAiEmbeddingResponse = serde_json::from_value(v).map_err(|e| {
- LlmError::Provider {
+ let parsed: OpenAiEmbeddingResponse =
+ serde_json::from_value(v).map_err(|e| LlmError::Provider {
status,
body: truncate_with_ellipsis(
&format!("openai embeddings shape: {e}; body={body}"),
1024,
),
- }
- })?;
+ })?;
let first = parsed.data.into_iter().next().ok_or_else(|| {
LlmError::UnexpectedShape(format!(
"openai response had no data[0]; body={}",
diff --git a/crates/ai-memory-llm/src/google.rs b/crates/ai-memory-llm/src/google.rs
index dddc9119..d1ca1cfd 100644
--- a/crates/ai-memory-llm/src/google.rs
+++ b/crates/ai-memory-llm/src/google.rs
@@ -97,7 +97,11 @@ impl GoogleEmbedder {
if status.as_u16() == 429 && attempt < 5 {
attempt += 1;
let delay = Duration::from_secs(2u64.saturating_pow(attempt));
- debug!(attempt, ?delay, "google embedContent rate-limited; retrying");
+ debug!(
+ attempt,
+ ?delay,
+ "google embedContent rate-limited; retrying"
+ );
tokio::time::sleep(delay).await;
continue;
}
diff --git a/crates/ai-memory-llm/src/text.rs b/crates/ai-memory-llm/src/text.rs
index b04d60c6..861ea357 100644
--- a/crates/ai-memory-llm/src/text.rs
+++ b/crates/ai-memory-llm/src/text.rs
@@ -35,12 +35,16 @@ pub(crate) fn suffix_within_bytes(s: &str, max_bytes: usize) -> &str {
/// both a token budget (optimistic prose) and a hard byte ceiling.
pub(crate) fn truncate_for_embedding(text: &str, max_tokens: usize) -> String {
const HARD_MAX_BYTES: usize = 8_000;
+ const ELLIPSIS_BYTES: usize = "…".len();
let token_budget_bytes = max_tokens.saturating_mul(3);
let max_bytes = token_budget_bytes.min(HARD_MAX_BYTES);
if text.len() <= max_bytes {
return text.to_string();
}
- truncate_with_ellipsis(text, max_bytes.saturating_sub(1))
+ if max_bytes <= ELLIPSIS_BYTES {
+ return String::new();
+ }
+ truncate_with_ellipsis(text, max_bytes - ELLIPSIS_BYTES)
}
#[cfg(test)]
@@ -73,6 +77,6 @@ mod tests {
let out = truncate_for_embedding(&long, 6000);
assert!(out.ends_with('…'));
assert!(out.len() < long.len());
- assert!(out.len() <= 8_000 + 4);
+ assert!(out.len() <= 8_000);
}
}
diff --git a/docs/install.md b/docs/install.md
index 1a37f114..49395bfa 100644
--- a/docs/install.md
+++ b/docs/install.md
@@ -357,7 +357,7 @@ If you set only the provider, ai-memory picks a sensible default:
| `AI_MEMORY_LLM_PROVIDER=openai` | `gpt-5.4-mini` | Cheaper + faster alternative. Same parse reliability; mild over-classification on thin sessions. |
| `AI_MEMORY_LLM_PROVIDER=gemini` | `gemini-2.5-flash` | Google's hosted option with a generous free tier. ai-memory disables Gemini 2.5 Flash's default dynamic thinking so hidden thought tokens do not truncate strict JSON. Set `GEMINI_API_KEY` (or `GOOGLE_API_KEY`). |
| `AI_MEMORY_EMBEDDING_PROVIDER=openai` | `text-embedding-3-small` (1536-dim) | 5× cheaper than `-3-large` with marginal recall loss. |
-| `AI_MEMORY_EMBEDDING_PROVIDER=openai` + `AI_MEMORY_EMBEDDING_BASE_URL=https://openrouter.ai/api/v1` | `openai/text-embedding-3-small` via [OpenRouter](https://openrouter.ai) | Same API key as `LLM_API_KEY` / `OPENAI_API_KEY`; no código extra além do cliente OpenAI-compat. |
+| `AI_MEMORY_EMBEDDING_PROVIDER=openai` + `AI_MEMORY_EMBEDDING_BASE_URL=https://openrouter.ai/api/v1` | `openai/text-embedding-3-small` via [OpenRouter](https://openrouter.ai) | Reuses `LLM_API_KEY` or `OPENAI_API_KEY` with the OpenAI-compatible embedding client. |
| `AI_MEMORY_EMBEDDING_PROVIDER=voyage` | `voyage-3` (1024-dim) | Voyage's current general-purpose recommendation. |
| `AI_MEMORY_EMBEDDING_PROVIDER=google` / `gemini` | `gemini-embedding-001` (768-dim) | Google-hosted embeddings via `embedContent`. Set `GEMINI_API_KEY` (or `GOOGLE_API_KEY`). |