Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

feat(llm): make the per-request provider timeout configurable

Lucas Oliveira00f11e3acaad
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1c8020b6..4edb8975 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Added
+- New `llm_timeout_secs` config key (env `AI_MEMORY_LLM_TIMEOUT_SECS`, or
+ `llm_timeout_secs = 900` in config.toml) overrides the per-request timeout
+ every chat provider applies to its HTTP calls — previously hardcoded at
+ 300s per provider. Slow hosted gateways (observed with free aggregator
+ tiers whose long completions exceed five minutes) failed every request with
+ `http: error sending request` once generation crossed the ceiling, so LLM
+ consolidation exhausted its retries and left heuristic pages behind; now
+ operators raise the bound at startup instead. The Copilot token exchange
+ is bounded by the same value, and the 300s default is unchanged ([#435]).
- New `strip_root_combinators` config flag (env `AI_MEMORY_STRIP_ROOT_COMBINATORS`,
or `strip_root_combinators = true` in config.toml) strips root-level
`anyOf`/`oneOf`/`allOf` from MCP tool input schemas on every `tools/list`.
diff --git a/README.md b/README.md
index 4b5c13a0..98f3d8d6 100644
--- a/README.md
+++ b/README.md
@@ -966,6 +966,13 @@ automatic PreCompact/PostCompaction checkpoint fall back to the deterministic
rule-based page; admission, storage, and scope errors still fail closed. The
validated minimums are 6,000 input and 1,000 output tokens.
+Every chat provider bounds each HTTP request at 300 seconds
+(`AI_MEMORY_LLM_TIMEOUT_SECS` to override, or `llm_timeout_secs = 900` in
+config.toml). The default tolerates a local engine cold-loading a large model;
+slow hosted gateways whose long completions exceed the ceiling fail every
+request with `http: error sending request`, so raise the value there instead
+of watching consolidation exhaust its retries.
+
Reranking is optional and off by default. With an LLM provider configured,
`AI_MEMORY_RERANKER=llm` makes project and explicit-scope `memory_query`
calls over-fetch from the hybrid stage, fuse scopes, and make at most one LLM
diff --git a/crates/ai-memory-cli/src/commands/llm_test.rs b/crates/ai-memory-cli/src/commands/llm_test.rs
index 140b0396..f1c4212a 100644
--- a/crates/ai-memory-cli/src/commands/llm_test.rs
+++ b/crates/ai-memory-cli/src/commands/llm_test.rs
@@ -24,6 +24,7 @@ pub async fn run(config: &Config, args: LlmTestArgs) -> Result<()> {
auth: config.provider_auth(provider, api_key_override),
base_url: args.base_url.or_else(|| config.llm_test_base_url()),
compat_strict: config.llm_compat_strict,
+ request_timeout_secs: config.llm_timeout_secs,
};
let client = build_provider(provider_config).context("building LLM provider")?;
info!(
diff --git a/crates/ai-memory-cli/src/config.rs b/crates/ai-memory-cli/src/config.rs
index 615c61fb..bce462ea 100644
--- a/crates/ai-memory-cli/src/config.rs
+++ b/crates/ai-memory-cli/src/config.rs
@@ -140,6 +140,14 @@ pub struct Config {
/// already supplies its own schema. Set
/// `AI_MEMORY_LLM_COMPAT_STRICT=false` for an incompatible endpoint.
pub llm_compat_strict: bool,
+ /// Per-request timeout (seconds) applied to every chat provider's
+ /// HTTP calls, including the Copilot token exchange. Defaults to
+ /// `ai_memory_llm::DEFAULT_REQUEST_TIMEOUT_SECS` (300s), which
+ /// tolerates a local engine cold-loading a large model. Raise it
+ /// for slow hosted gateways whose long completions exceed the
+ /// ceiling (observed with free aggregator tiers). Set with
+ /// `AI_MEMORY_LLM_TIMEOUT_SECS`.
+ pub llm_timeout_secs: u64,
/// Opt-in: run LLM consolidation on SessionEnd (in addition to the
/// always-written heuristic session page), when an LLM provider is
/// configured. Off by default. Provider work is durably queued after the
@@ -552,6 +560,7 @@ impl Default for Config {
llm_model: None,
llm_base_url: None,
llm_compat_strict: true,
+ llm_timeout_secs: ai_memory_llm::DEFAULT_REQUEST_TIMEOUT_SECS,
consolidate_on_session_end: false,
capture_assistant: false,
strip_root_combinators: false,
@@ -919,6 +928,15 @@ impl Config {
config.consolidation.max_output_tokens
);
}
+ // Zero (or a sub-second remainder rounded down) would cut every
+ // provider request off before it is sent.
+ if config.llm_timeout_secs == 0 {
+ anyhow::bail!(
+ "llm_timeout_secs must be at least 1 second (got {}); \
+ AI_MEMORY_LLM_TIMEOUT_SECS is read in seconds",
+ config.llm_timeout_secs
+ );
+ }
Ok(config)
}
@@ -985,6 +1003,7 @@ impl Config {
.clone()
.or_else(|| self.runtime_env.llm_base_url.clone()),
compat_strict: self.llm_compat_strict,
+ request_timeout_secs: self.llm_timeout_secs,
}))
}
@@ -1303,6 +1322,10 @@ mod tests {
assert_eq!(cfg.bind, DEFAULT_BIND);
assert_eq!(cfg.server_url, DEFAULT_SERVER_URL);
assert_eq!(cfg.log_level, "info");
+ assert_eq!(
+ cfg.llm_timeout_secs,
+ ai_memory_llm::DEFAULT_REQUEST_TIMEOUT_SECS
+ );
assert!(!cfg.auth.secure_cookie);
assert!(cfg.maintenance.enabled);
assert_eq!(cfg.maintenance.forget_sweep_interval_secs, 86_400);
@@ -1439,6 +1462,32 @@ mod tests {
assert_eq!(cfg.consolidation.max_output_tokens, 1_000);
}
+ /// `AI_MEMORY_LLM_TIMEOUT_SECS` (figment maps it to this field) exists so
+ /// slow hosted gateways can outlive the 300s default; the accepted floor
+ /// mirrors that motivation — anything below one second severs every
+ /// provider request before it is sent.
+ #[test]
+ fn load_round_trips_a_custom_llm_request_timeout() {
+ let tmp = TempDir::new().unwrap();
+ let config_path = tmp.path().join("config.toml");
+ std::fs::write(&config_path, "llm_timeout_secs = 900\n").unwrap();
+ let cfg = Config::load(Some(&config_path), Some(tmp.path().to_path_buf())).unwrap();
+ assert_eq!(cfg.llm_timeout_secs, 900);
+ }
+
+ #[test]
+ fn load_rejects_a_zero_llm_request_timeout() {
+ let tmp = TempDir::new().unwrap();
+ let config_path = tmp.path().join("config.toml");
+ std::fs::write(&config_path, "llm_timeout_secs = 0\n").unwrap();
+ let error = Config::load(Some(&config_path), Some(tmp.path().to_path_buf()))
+ .expect_err("a sub-second timeout must fail closed");
+ assert!(
+ error.to_string().contains("llm_timeout_secs"),
+ "unexpected error: {error:#}"
+ );
+ }
+
#[test]
fn defaults_bound_consolidation_prompts_for_a_large_context_provider() {
let tmp = TempDir::new().unwrap();
diff --git a/crates/ai-memory-llm/src/anthropic.rs b/crates/ai-memory-llm/src/anthropic.rs
index a04aea5d..a1bd67c8 100644
--- a/crates/ai-memory-llm/src/anthropic.rs
+++ b/crates/ai-memory-llm/src/anthropic.rs
@@ -45,6 +45,7 @@ pub struct AnthropicProvider {
auth: AnthropicAuth,
base_url: String,
model: String,
+ timeout: Duration,
}
impl AnthropicProvider {
@@ -54,17 +55,13 @@ impl AnthropicProvider {
/// Returns a `reqwest::Error` if the underlying HTTP client cannot
/// be built.
pub fn new(api_key: SecretString, model: impl Into<String>) -> LlmResult<Self> {
- // 300s matches the OpenAI/openai-compat client — same reason:
- // first request after a model swap on a local inference server
- // (Ollama, llama-swap, vLLM) can take 30-90s of cold-load.
- let client = reqwest::Client::builder()
- .timeout(Duration::from_secs(300))
- .build()?;
+ let client = reqwest::Client::builder().build()?;
Ok(Self {
client,
auth: AnthropicAuth::ApiKey(api_key),
base_url: DEFAULT_BASE_URL.to_string(),
model: model.into(),
+ timeout: Duration::from_secs(crate::DEFAULT_REQUEST_TIMEOUT_SECS),
})
}
@@ -78,14 +75,13 @@ impl AnthropicProvider {
/// Returns a `reqwest::Error` if the underlying HTTP client cannot
/// be built.
pub fn new_oauth(token: SecretString, model: impl Into<String>) -> LlmResult<Self> {
- let client = reqwest::Client::builder()
- .timeout(Duration::from_secs(300))
- .build()?;
+ let client = reqwest::Client::builder().build()?;
Ok(Self {
client,
auth: AnthropicAuth::OAuth(token),
base_url: DEFAULT_BASE_URL.to_string(),
model: model.into(),
+ timeout: Duration::from_secs(crate::DEFAULT_REQUEST_TIMEOUT_SECS),
})
}
@@ -95,6 +91,14 @@ impl AnthropicProvider {
self.base_url = url.into();
self
}
+
+ /// Override the per-request timeout (default
+ /// [`crate::DEFAULT_REQUEST_TIMEOUT_SECS`]).
+ #[must_use]
+ pub fn with_timeout_secs(mut self, secs: u64) -> Self {
+ self.timeout = Duration::from_secs(secs);
+ self
+ }
}
#[derive(Debug, Serialize)]
@@ -260,6 +264,7 @@ impl AnthropicProvider {
let mut builder = self
.client
.post(&url)
+ .timeout(self.timeout)
.header("anthropic-version", ANTHROPIC_VERSION)
.header("content-type", "application/json");
// Apply the auth headers through the same helper the tests assert on,
diff --git a/crates/ai-memory-llm/src/copilot.rs b/crates/ai-memory-llm/src/copilot.rs
index 97ce2e4e..826840e0 100644
--- a/crates/ai-memory-llm/src/copilot.rs
+++ b/crates/ai-memory-llm/src/copilot.rs
@@ -185,6 +185,7 @@ pub struct CopilotProvider {
model: String,
auth: CopilotAuth,
stored: Mutex<CopilotToken>,
+ timeout: Duration,
}
impl CopilotProvider {
@@ -206,7 +207,6 @@ impl CopilotProvider {
)));
}
let client = reqwest::Client::builder()
- .timeout(Duration::from_secs(300))
.user_agent(COPILOT_USER_AGENT)
.build()
.map_err(LlmError::from)?;
@@ -215,9 +215,19 @@ impl CopilotProvider {
model: model.into(),
auth,
stored: Mutex::new(stored),
+ timeout: Duration::from_secs(crate::DEFAULT_REQUEST_TIMEOUT_SECS),
})
}
+ /// Override the per-request timeout (default
+ /// [`crate::DEFAULT_REQUEST_TIMEOUT_SECS`]). Also bounds the
+ /// GitHub token-exchange request.
+ #[must_use]
+ pub fn with_timeout_secs(mut self, secs: u64) -> Self {
+ self.timeout = Duration::from_secs(secs);
+ self
+ }
+
async fn current_token(&self) -> LlmResult<CopilotApiToken> {
if let Some(token) = self.auth.direct_api_token.as_ref() {
return Ok(CopilotApiToken {
@@ -251,7 +261,7 @@ impl CopilotProvider {
})?;
info!("copilot API token expired or missing, exchanging GitHub token");
- let exchanged = exchange_copilot_token(&self.client, &github).await?;
+ let exchanged = exchange_copilot_token(&self.client, self.timeout, &github).await?;
guard.copilot_access = Some(exchanged.access.clone());
guard.copilot_expires_at_ms = Some(exchanged.expires_at_ms);
guard.api_base_url = Some(resolve_copilot_api_base_url(
@@ -277,6 +287,7 @@ impl CopilotProvider {
let resp = self
.client
.post(&url)
+ .timeout(self.timeout)
.bearer_auth(token.access.expose_secret())
.headers(copilot_runtime_headers())
.json(body)
@@ -356,10 +367,12 @@ struct CopilotExchangeResponse {
async fn exchange_copilot_token(
client: &reqwest::Client,
+ timeout: Duration,
github_token: &SecretString,
) -> LlmResult<CopilotApiToken> {
let resp = client
.get(GITHUB_COPILOT_TOKEN_URL)
+ .timeout(timeout)
.bearer_auth(github_token.expose_secret())
.headers(copilot_token_exchange_headers())
.send()
diff --git a/crates/ai-memory-llm/src/factory.rs b/crates/ai-memory-llm/src/factory.rs
index 37ac2db3..90130840 100644
--- a/crates/ai-memory-llm/src/factory.rs
+++ b/crates/ai-memory-llm/src/factory.rs
@@ -99,6 +99,10 @@ pub struct ProviderConfig {
/// parser. Enabled by default and ignored by every other provider.
/// Sourced once from `AI_MEMORY_LLM_COMPAT_STRICT` by `Config::load`.
pub compat_strict: bool,
+ /// Per-request timeout for every chat provider, in seconds.
+ /// Sourced once from `AI_MEMORY_LLM_TIMEOUT_SECS` by `Config::load`;
+ /// defaults to [`crate::DEFAULT_REQUEST_TIMEOUT_SECS`].
+ pub request_timeout_secs: u64,
}
/// Embedding providers available to ai-memory.
@@ -227,18 +231,25 @@ pub fn try_default_embedding_dim(provider: EmbedderChoice, model: &str) -> Optio
/// Returns [`LlmError::NotConfigured`] if a required env value (API
/// key, base URL) is missing.
pub fn build_provider(config: ProviderConfig) -> LlmResult<Arc<dyn LlmProvider>> {
+ let timeout = config.request_timeout_secs;
match config.provider {
ProviderChoice::Anthropic => {
let key = config.auth.require_api_key()?;
- Ok(Arc::new(AnthropicProvider::new(key, config.model)?))
+ Ok(Arc::new(
+ AnthropicProvider::new(key, config.model)?.with_timeout_secs(timeout),
+ ))
}
ProviderChoice::OpenAi => {
let key = config.auth.require_api_key()?;
- Ok(Arc::new(OpenAiProvider::new(key, config.model)?))
+ Ok(Arc::new(
+ OpenAiProvider::new(key, config.model)?.with_timeout_secs(timeout),
+ ))
}
ProviderChoice::Gemini => {
let key = config.auth.require_api_key()?;
- Ok(Arc::new(GeminiProvider::new(key, config.model)?))
+ Ok(Arc::new(
+ GeminiProvider::new(key, config.model)?.with_timeout_secs(timeout),
+ ))
}
ProviderChoice::OpenAiCompat => {
let base = config
@@ -246,16 +257,21 @@ pub fn build_provider(config: ProviderConfig) -> LlmResult<Arc<dyn LlmProvider>>
.ok_or_else(|| LlmError::NotConfigured("LLM_BASE_URL".into()))?;
Ok(Arc::new(
OpenAiCompatProvider::new(base, config.auth.optional_api_key(), config.model)?
- .with_strict(config.compat_strict),
+ .with_strict(config.compat_strict)
+ .with_timeout_secs(timeout),
))
}
ProviderChoice::OpenAiOAuth => {
let path = config.auth.require_openai_oauth_token_file()?.to_path_buf();
- Ok(Arc::new(OpenAiOAuthProvider::new(path, config.model)?))
+ Ok(Arc::new(
+ OpenAiOAuthProvider::new(path, config.model)?.with_timeout_secs(timeout),
+ ))
}
ProviderChoice::Copilot => {
let auth = config.auth.require_copilot_auth()?;
- Ok(Arc::new(CopilotProvider::new(auth, config.model)?))
+ Ok(Arc::new(
+ CopilotProvider::new(auth, config.model)?.with_timeout_secs(timeout),
+ ))
}
ProviderChoice::AnthropicOAuth => {
let token = config.auth.require_anthropic_oauth_token()?;
@@ -263,11 +279,13 @@ pub fn build_provider(config: ProviderConfig) -> LlmResult<Arc<dyn LlmProvider>>
if let Some(url) = config.base_url {
provider = provider.with_base_url(url);
}
- Ok(Arc::new(provider))
+ Ok(Arc::new(provider.with_timeout_secs(timeout)))
}
ProviderChoice::OpenCode => {
let key = config.auth.require_api_key()?;
- Ok(Arc::new(OpenCodeProvider::new(key, config.model)?))
+ Ok(Arc::new(
+ OpenCodeProvider::new(key, config.model)?.with_timeout_secs(timeout),
+ ))
}
}
}
@@ -324,6 +342,7 @@ mod tests {
auth: ProviderAuth::required_api_key_from_env("OPENAI_API_KEY", None),
base_url: None,
compat_strict: false,
+ request_timeout_secs: crate::DEFAULT_REQUEST_TIMEOUT_SECS,
};
let err = match build_provider(cfg) {
diff --git a/crates/ai-memory-llm/src/gemini.rs b/crates/ai-memory-llm/src/gemini.rs
index c3973dad..502dcd5e 100644
--- a/crates/ai-memory-llm/src/gemini.rs
+++ b/crates/ai-memory-llm/src/gemini.rs
@@ -30,6 +30,7 @@ pub struct GeminiProvider {
api_key: SecretString,
base_url: String,
model: String,
+ timeout: Duration,
}
impl GeminiProvider {
@@ -39,14 +40,13 @@ impl GeminiProvider {
/// # Errors
/// Returns a `reqwest::Error` if the HTTP client cannot be built.
pub fn new(api_key: SecretString, model: impl Into<String>) -> LlmResult<Self> {
- let client = reqwest::Client::builder()
- .timeout(Duration::from_secs(300))
- .build()?;
+ let client = reqwest::Client::builder().build()?;
Ok(Self {
client,
api_key,
base_url: DEFAULT_BASE_URL.to_string(),
model: model.into(),
+ timeout: Duration::from_secs(crate::DEFAULT_REQUEST_TIMEOUT_SECS),
})
}
@@ -56,6 +56,14 @@ impl GeminiProvider {
self.base_url = url.into();
self
}
+
+ /// Override the per-request timeout (default
+ /// [`crate::DEFAULT_REQUEST_TIMEOUT_SECS`]).
+ #[must_use]
+ pub fn with_timeout_secs(mut self, secs: u64) -> Self {
+ self.timeout = Duration::from_secs(secs);
+ self
+ }
}
#[derive(Debug, Serialize)]
@@ -228,6 +236,7 @@ impl GeminiProvider {
let resp = self
.client
.post(&url)
+ .timeout(self.timeout)
.header("x-goog-api-key", self.api_key.expose_secret())
.header("content-type", "application/json")
.json(body)
diff --git a/crates/ai-memory-llm/src/lib.rs b/crates/ai-memory-llm/src/lib.rs
index f6ed6e91..2166a9e0 100644
--- a/crates/ai-memory-llm/src/lib.rs
+++ b/crates/ai-memory-llm/src/lib.rs
@@ -26,6 +26,16 @@
//! otherwise parse the first balanced `{…}` from the text body.
//! No tenacity-style 8-128s backoff (cognee #2840 lesson).
+/// Default per-request timeout applied by every chat provider.
+///
+/// 300s tolerates Ollama / llama-swap cold-loading a 30B+ model from disk
+/// on first request. Once `OLLAMA_KEEP_ALIVE` keeps it warm, subsequent
+/// requests return in seconds — but the first one after the model unloaded
+/// needs the headroom. Slow hosted gateways that stream long completions
+/// may need more; operators override it with `AI_MEMORY_LLM_TIMEOUT_SECS`
+/// (read once by `Config::load`, applied to every chat provider).
+pub const DEFAULT_REQUEST_TIMEOUT_SECS: u64 = 300;
+
pub mod anthropic;
pub mod auth;
pub mod copilot;
diff --git a/crates/ai-memory-llm/src/openai.rs b/crates/ai-memory-llm/src/openai.rs
index 26fa3a21..da1e6b16 100644
--- a/crates/ai-memory-llm/src/openai.rs
+++ b/crates/ai-memory-llm/src/openai.rs
@@ -89,6 +89,7 @@ pub struct OpenAiProvider {
base_url: String,
model: String,
dialect: RequestDialect,
+ timeout: Duration,
}
impl OpenAiProvider {
@@ -99,19 +100,14 @@ impl OpenAiProvider {
/// # Errors
/// Returns a `reqwest::Error` if the HTTP client cannot be built.
pub fn new(api_key: SecretString, model: impl Into<String>) -> LlmResult<Self> {
- // 300s tolerates Ollama / llama-swap cold-loading a 30B+ model
- // from disk on first request. Once OLLAMA_KEEP_ALIVE keeps it
- // warm, subsequent requests return in seconds — but the first
- // one after the model unloaded needs the headroom.
- let client = reqwest::Client::builder()
- .timeout(Duration::from_secs(300))
- .build()?;
+ let client = reqwest::Client::builder().build()?;
Ok(Self {
client,
api_key,
base_url: DEFAULT_BASE_URL.to_string(),
model: model.into(),
dialect: RequestDialect::Official,
+ timeout: Duration::from_secs(crate::DEFAULT_REQUEST_TIMEOUT_SECS),
})
}
@@ -123,6 +119,23 @@ impl OpenAiProvider {
self
}
+ /// Override the per-request timeout (default
+ /// [`crate::DEFAULT_REQUEST_TIMEOUT_SECS`]). Applied per request,
+ /// so the HTTP client itself stays connection-pool friendly.
+ #[must_use]
+ pub fn with_timeout_secs(mut self, secs: u64) -> Self {
+ self.timeout = Duration::from_secs(secs);
+ self
+ }
+
+ /// Currently configured per-request timeout. Test-visible so
+ /// wrapper tests (`OpenAiCompatProvider`, `OpenCodeProvider`)
+ /// can assert the delegation without exposing the field.
+ #[cfg(test)]
+ pub(crate) fn request_timeout(&self) -> Duration {
+ self.timeout
+ }
+
/// Switch request dialect. See [`RequestDialect`].
#[must_use]
pub fn with_dialect(mut self, dialect: RequestDialect) -> Self {
@@ -316,6 +329,7 @@ impl OpenAiProvider {
let resp = self
.client
.post(&url)
+ .timeout(self.timeout)
.bearer_auth(self.api_key.expose_secret())
.header("content-type", "application/json")
.json(body)
@@ -480,6 +494,17 @@ mod tests {
OpenAiProvider::new(SecretString::new("test-key".into()), model).unwrap()
}
+ #[test]
+ fn request_timeout_defaults_and_is_overridable() {
+ let provider = provider_for("gpt-4o-mini");
+ assert_eq!(
+ provider.timeout,
+ std::time::Duration::from_secs(crate::DEFAULT_REQUEST_TIMEOUT_SECS)
+ );
+ let provider = provider.with_timeout_secs(900);
+ assert_eq!(provider.timeout, std::time::Duration::from_secs(900));
+ }
+
fn chat_request() -> ChatRequest {
ChatRequest {
system: None,
diff --git a/crates/ai-memory-llm/src/openai_compat.rs b/crates/ai-memory-llm/src/openai_compat.rs
index 3bed1122..21c63e2c 100644
--- a/crates/ai-memory-llm/src/openai_compat.rs
+++ b/crates/ai-memory-llm/src/openai_compat.rs
@@ -98,6 +98,15 @@ impl OpenAiCompatProvider {
self.strict = strict;
self
}
+
+ /// Override the per-request timeout on the wrapped
+ /// [`OpenAiProvider`]. The factory calls this with
+ /// `ProviderConfig::request_timeout_secs`.
+ #[must_use]
+ pub fn with_timeout_secs(mut self, secs: u64) -> Self {
+ self.inner = self.inner.with_timeout_secs(secs);
+ self
+ }
}
#[async_trait]
@@ -288,6 +297,17 @@ mod tests {
assert!(p.strict);
}
+ #[test]
+ fn timeout_override_reaches_the_inner_provider() {
+ let p = OpenAiCompatProvider::new("http://localhost:11434/v1", None, "mistral-nemo")
+ .expect("provider builds")
+ .with_timeout_secs(45);
+ assert_eq!(
+ p.inner.request_timeout(),
+ std::time::Duration::from_secs(45)
+ );
+ }
+
#[test]
fn only_explicit_response_format_rejections_are_retryable() {
assert!(is_response_format_rejection(&LlmError::Provider {
diff --git a/crates/ai-memory-llm/src/openai_oauth.rs b/crates/ai-memory-llm/src/openai_oauth.rs
index bd9c136c..a78b1d13 100644
--- a/crates/ai-memory-llm/src/openai_oauth.rs
+++ b/crates/ai-memory-llm/src/openai_oauth.rs
@@ -138,6 +138,7 @@ pub struct OpenAiOAuthProvider {
model: String,
token_path: PathBuf,
token: Mutex<OpenAiOAuthToken>,
+ timeout: Duration,
}
impl OpenAiOAuthProvider {
@@ -152,18 +153,24 @@ impl OpenAiOAuthProvider {
token_path.display()
))
})?;
- let client = reqwest::Client::builder()
- .timeout(Duration::from_secs(300))
- .build()
- .map_err(LlmError::from)?;
+ let client = reqwest::Client::builder().build().map_err(LlmError::from)?;
Ok(Self {
client,
model: model.into(),
token_path,
token: Mutex::new(token),
+ timeout: Duration::from_secs(crate::DEFAULT_REQUEST_TIMEOUT_SECS),
})
}
+ /// Override the per-request timeout (default
+ /// [`crate::DEFAULT_REQUEST_TIMEOUT_SECS`]).
+ #[must_use]
+ pub fn with_timeout_secs(mut self, secs: u64) -> Self {
+ self.timeout = Duration::from_secs(secs);
+ self
+ }
+
async fn current_token(&self) -> LlmResult<OpenAiOAuthToken> {
let mut guard = self.token.lock().await;
if guard.needs_refresh() {
@@ -184,6 +191,7 @@ impl OpenAiOAuthProvider {
let mut request = self
.client
.post(CODEX_RESPONSES_URL)
+ .timeout(self.timeout)
.bearer_auth(token.access.expose_secret())
.header("content-type", "application/json")
.header(
diff --git a/crates/ai-memory-llm/src/opencode.rs b/crates/ai-memory-llm/src/opencode.rs
index 301241aa..f1aa10c0 100644
--- a/crates/ai-memory-llm/src/opencode.rs
+++ b/crates/ai-memory-llm/src/opencode.rs
@@ -36,6 +36,15 @@ impl OpenCodeProvider {
let inner = OpenAiCompatProvider::new(OPENCODE_ZEN_BASE_URL, Some(api_key), model.into())?;
Ok(Self { inner })
}
+
+ /// Override the per-request timeout on the wrapped
+ /// [`OpenAiCompatProvider`]. The factory calls this with
+ /// `ProviderConfig::request_timeout_secs`.
+ #[must_use]
+ pub fn with_timeout_secs(mut self, secs: u64) -> Self {
+ self.inner = self.inner.with_timeout_secs(secs);
+ self
+ }
}
#[async_trait]
diff --git a/crates/ai-memory-llm/src/stored_token.rs b/crates/ai-memory-llm/src/stored_token.rs
index 946cb150..41f95853 100644
--- a/crates/ai-memory-llm/src/stored_token.rs
+++ b/crates/ai-memory-llm/src/stored_token.rs
@@ -7,6 +7,7 @@
//! provider only defines its extras and its response mapping.
use std::path::Path;
+use std::time::Duration;
use secrecy::{ExposeSecret as _, SecretString};
use serde::{Deserialize, Serialize, de::DeserializeOwned};
@@ -126,6 +127,10 @@ where
{
let resp = client
.post(token_endpoint)
+ // Bounded here because callers hand us their shared client, whose
+ // global timeout was removed in favour of per-request timeouts —
+ // a token exchange is quick, so the default ceiling is plenty.
+ .timeout(Duration::from_secs(crate::DEFAULT_REQUEST_TIMEOUT_SECS))
.form(&[
("grant_type", "refresh_token"),
("refresh_token", refresh_token),
diff --git a/docs/deploy.md b/docs/deploy.md
index 26c97d04..1354b6e3 100644
--- a/docs/deploy.md
+++ b/docs/deploy.md
@@ -180,6 +180,12 @@ tolerant fallback for explicit capability rejection or malformed output. Set
`AI_MEMORY_LLM_COMPAT_STRICT=false` for an incompatible endpoint. If you switch
to a niche local model, run a quick `ai-memory llm-test` before trusting it.
+Every chat provider bounds each HTTP request at 300 seconds. Slow hosted
+gateways (observed with free aggregator tiers) can stream a long completion
+past that ceiling and fail every request with `http: error sending request`;
+raise `AI_MEMORY_LLM_TIMEOUT_SECS` in the container environment to match the
+gateway's worst-case generation time.
+
## Backups
The data dir is whatever you mounted in `docker-compose.prod.yml`
diff --git a/evals/src/main.rs b/evals/src/main.rs
index 23a214ed..5c00b4f7 100644
--- a/evals/src/main.rs
+++ b/evals/src/main.rs
@@ -371,6 +371,7 @@ impl From<ResolvedConfig> for ProviderConfig {
// Match the product default so provider comparisons exercise the
// same schema-constrained path operators receive.
compat_strict: true,
+ request_timeout_secs: ai_memory_llm::DEFAULT_REQUEST_TIMEOUT_SECS,
}
}
}