World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
A rate limit gets waited out, not skipped (#160)
WaylandYang3d9c0ae9000c
diff --git a/crates/utopia-llm/src/lib.rs b/crates/utopia-llm/src/lib.rsindex 7ec91636..5e182114 100644--- a/crates/utopia-llm/src/lib.rs+++ b/crates/utopia-llm/src/lib.rs@@ -83,6 +83,60 @@ pub fn is_unreachable(err: &anyhow::Error) -> bool {err.chain().any(|e| e.is::<Unreachable>())}+/// 端点在限流。**跟 [`Unreachable`] 一样做成类型**,理由也一样:调用方要据此+/// 决定「等一会儿再来」而不是「这块废了」,而错误文本一路都在被 context 改写。+///+/// 限流与其他 4xx 的区别是**它会自己好**。密钥错了重试一万次还是错,配额满了+/// 等一分钟就过去——两者混在一起,重试预算就会花在永远不会好的那一类上。+#[derive(Debug, thiserror::Error)]+#[error("LLM endpoint is rate limiting ({status}): {detail}")]+pub struct RateLimited {+ pub status: u16,+ /// **常常是 `None`。** 多数厂商的 429 不带 `Retry-After`(实测 SiliconFlow+ /// 就不带),所以调用方必须自带退避,把这一项当「有则更准」的补充而不是判据。+ pub retry_after: Option<Duration>,+ pub detail: String,+}++/// anyhow 错误链里的 [`RateLimited`],穿透 context 层。+pub fn rate_limited(err: &anyhow::Error) -> Option<&RateLimited> {+ err.chain().find_map(|e| e.downcast_ref::<RateLimited>())+}++/// `Retry-After` 的整数秒形态。+///+/// 规范还允许 HTTP-date,这里**不解析**:为一个很少有人发的头引一个日期库不划算,+/// 而解析失败当成没有正是对的——调用方本来就得有退避,多猜一个数只会更难查。+fn retry_after_of(headers: &reqwest::header::HeaderMap) -> Option<Duration> {+ headers+ .get(reqwest::header::RETRY_AFTER)?+ .to_str()+ .ok()?+ .trim()+ .parse::<u64>()+ .ok()+ .map(Duration::from_secs)+}++/// 非 2xx 统一在这里成型。**只有 429 算限流**:503 可能是端点真挂了、+/// 也可能是中间代理,把它算进来会让「等一会儿再来」用在等不回来的地方。+fn failure(+ kind: &str,+ status: reqwest::StatusCode,+ retry_after: Option<Duration>,+ body: &serde_json::Value,+) -> anyhow::Error {+ let detail = err_detail(body);+ if status == reqwest::StatusCode::TOO_MANY_REQUESTS {+ return anyhow::Error::new(RateLimited {+ status: status.as_u16(),+ retry_after,+ detail,+ });+ }+ anyhow::anyhow!("{kind} request failed ({status}): {detail}")+}+#[derive(Clone)]pub struct LlmClient {http: reqwest::Client,@@ -159,9 +213,10 @@ impl LlmClient {.await.map_err(Unreachable)?;let status = resp.status();+ let retry_after = retry_after_of(resp.headers());let body: serde_json::Value = resp.json().await.map_err(Unreachable)?;if !status.is_success() {- anyhow::bail!("LLM request failed ({status}): {}", err_detail(&body));+ return Err(failure("LLM", status, retry_after, &body));}log_usage(&self.model, &body);body["choices"][0]["message"]["content"]@@ -189,9 +244,10 @@ impl LlmClient {.await.map_err(Unreachable)?;let status = resp.status();+ let retry_after = retry_after_of(resp.headers());let body: serde_json::Value = resp.json().await.map_err(Unreachable)?;if !status.is_success() {- anyhow::bail!("LLM request failed ({status}): {}", err_detail(&body));+ return Err(failure("LLM", status, retry_after, &body));}let msg = &body["choices"][0]["message"];if msg.is_null() {@@ -245,8 +301,9 @@ impl LlmClient {.map_err(Unreachable)?;if !resp.status().is_success() {let status = resp.status();+ let retry_after = retry_after_of(resp.headers());let body: serde_json::Value = resp.json().await.unwrap_or_default();- anyhow::bail!("LLM request failed ({status}): {}", err_detail(&body));+ return Err(failure("LLM", status, retry_after, &body));}let mut bytes = resp.bytes_stream();@@ -337,8 +394,9 @@ impl LlmClient {.map_err(Unreachable)?;if !resp.status().is_success() {let status = resp.status();+ let retry_after = retry_after_of(resp.headers());let body: serde_json::Value = resp.json().await.unwrap_or_default();- anyhow::bail!("LLM request failed ({status}): {}", err_detail(&body));+ return Err(failure("LLM", status, retry_after, &body));}let mut bytes = resp.bytes_stream();@@ -381,9 +439,10 @@ impl LlmClient {.await.map_err(Unreachable)?;let status = resp.status();+ let retry_after = retry_after_of(resp.headers());let body: serde_json::Value = resp.json().await.map_err(Unreachable)?;if !status.is_success() {- anyhow::bail!("Embedding request failed ({status}): {}", err_detail(&body));+ return Err(failure("Embedding", status, retry_after, &body));}let data = body["data"].as_array()@@ -527,6 +586,62 @@ mod tests {assert!(!is_unreachable(&e));}+ /// **限流要穿透 context 层被认出来。**+ ///+ /// 与 [`unreachable_survives_context_layers`] 同一个理由,后果更重:认不出来+ /// 就退回「这块废了」,而限流本来一分钟后就过去了。实测一次 1884 块的灌入里+ /// 55/60 篇文档因此整篇失败。+ #[tokio::test]+ async fn a_rate_limit_survives_context_layers() {+ let err = anyhow::Error::new(RateLimited {+ status: 429,+ retry_after: None,+ detail: "TPM limit reached".into(),+ })+ .context("抽取失败")+ .context("process_document 失败");+ let hit = rate_limited(&err).expect("限流没被认出来");+ assert_eq!(hit.status, 429);+ assert!(!err.to_string().contains("rate limiting"));+ }++ /// **没有 `Retry-After` 是常规情况,不是异常。**+ ///+ /// 多数厂商的 429 不带这个头。判定必须只看类型,退避由调用方自己出——+ /// 把 `retry_after.is_some()` 当判据的话,这些厂商一个都识别不了。+ #[tokio::test]+ async fn a_rate_limit_without_retry_after_is_still_a_rate_limit() {+ let err = anyhow::Error::new(RateLimited {+ status: 429,+ retry_after: None,+ detail: "TPM limit reached".into(),+ });+ assert!(rate_limited(&err).is_some());+ assert!(rate_limited(&err).unwrap().retry_after.is_none());+ }++ /// 反面:限流不该被当成端点不可达,两者的处置完全不同。+ #[tokio::test]+ async fn a_rate_limit_is_not_an_unreachable_endpoint() {+ let err = anyhow::Error::new(RateLimited {+ status: 429,+ retry_after: Some(Duration::from_secs(7)),+ detail: "slow down".into(),+ });+ assert!(!is_unreachable(&err));+ assert_eq!(+ rate_limited(&err).unwrap().retry_after,+ Some(Duration::from_secs(7))+ );+ }++ /// 反面:别的 4xx 不是限流。密钥错了重试一万次还是错。+ #[tokio::test]+ async fn an_auth_failure_is_not_a_rate_limit() {+ let e = anyhow::anyhow!("LLM request failed (401 Unauthorized): bad key");+ assert!(rate_limited(&e).is_none());+ }+/// 端点干干净净地回了 4xx 不算——那说明它就是模型 API,/// 只是密钥或模型名不对,该找的人和该做的事都不一样。#[tokio::test]diff --git a/crates/utopia-server/src/extraction.rs b/crates/utopia-server/src/extraction.rsindex fcfe2238..19b06fac 100644--- a/crates/utopia-server/src/extraction.rs+++ b/crates/utopia-server/src/extraction.rs@@ -6,8 +6,76 @@ use crate::llm_util;use crate::predicate_match::PredicateIndex;use crate::state::AppState;use std::collections::{HashMap, HashSet};+use std::time::Duration;use uuid::Uuid;+/// 限流最多退避重试几次。**只对 429 生效**:密钥错了重试一万次还是错。+const RATE_LIMIT_TRIES: u32 = 5;+/// 单次退避的上限。总等待因此封顶在两分钟出头,一个配额永远打满的账号+/// 会干脆地失败,而不是把 worker 槽占死。+const RATE_LIMIT_CAP: Duration = Duration::from_secs(60);++/// 退避的抖动。**不引 `rand`**:这里只要「别让 N 个分块同时醒来」,纳秒时钟+/// 就够散,而多一个依赖要跟着走供应链。+///+/// 取半区间(base/2 到 base)而不是全区间:退避仍然单调增长,只是错开。+fn jitter(base: Duration) -> Duration {+ let nanos = std::time::SystemTime::now()+ .duration_since(std::time::UNIX_EPOCH)+ .map(|d| u64::from(d.subsec_nanos()))+ .unwrap_or(0);+ let half = (base.as_millis() as u64) / 2;+ base / 2 + Duration::from_millis(if half == 0 { 0 } else { nanos % half })+}++/// 抽取的 chat 调用,**限流会退避重试**。+///+/// 限流与其他失败的区别是它会自己好,所以从前那句「跳过该分块」用在它身上+/// 就是把一分钟的等待换成永久的数据缺口——实测一次 1884 块的灌入里+/// 55/60 篇文档整篇失败,而端点一直是好的。+///+/// 两处容易写错:+///+/// - **退避期间不能占着许可。** 许可只包住真正的调用,睡觉之前就还回去,+/// 否则一个在等的分块会挡住本来可以通过的另一个。+/// - **`Retry-After` 多数厂商不发**,所以它只是「有则更准」,判据是错误类型+/// 本身;没有它就走指数退避。+async fn chat_retrying_rate_limits(+ state: &AppState,+ settings: &utopia_core::models::LlmSettings,+ client: &utopia_llm::LlmClient,+ messages: &[utopia_llm::ChatMessage],+) -> anyhow::Result<String> {+ let mut backoff = Duration::from_secs(2);+ for attempt in 1..=RATE_LIMIT_TRIES {+ // 许可只包住调用本身,出了这个块就还回去+ let outcome = {+ let _permit = llm_util::acquire_chat(state, settings).await;+ client.chat(messages).await+ };+ let err = match outcome {+ Ok(reply) => return Ok(reply),+ Err(e) => e,+ };+ let Some(hit) = utopia_llm::rate_limited(&err) else {+ return Err(err);+ };+ if attempt == RATE_LIMIT_TRIES {+ return Err(err.context(format!("限流退避 {RATE_LIMIT_TRIES} 次仍未通过")));+ }+ let delay = jitter(hit.retry_after.unwrap_or(backoff).min(RATE_LIMIT_CAP));+ tracing::warn!(+ attempt,+ delay_ms = delay.as_millis() as u64,+ from_header = hit.retry_after.is_some(),+ "端点限流,退避后重试"+ );+ tokio::time::sleep(delay).await;+ backoff = (backoff * 2).min(RATE_LIMIT_CAP);+ }+ unreachable!("循环内必定 return")+}+const MIN_CONFIDENCE: f32 = 0.6;/// 这串字**是不是一个东西的名字**。@@ -374,11 +442,9 @@ async fn run(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {&known,&chunk.text,);- // 按模型限流:许可持有到调用结束,超出限额的分块在这里排队而不是打爆供应商- let _permit = llm_util::acquire_chat(state, &settings).await;// 这两处 continue 跳过的是**整个分块**——它一条事实都没产出。// 记下来,收尾时据此决定这篇文档算不算抽完(见循环之后)- let reply = match client.chat(&messages).await {+ let reply = match chat_retrying_rate_limits(state, &settings, &client, &messages).await {Ok(r) => r,Err(e) => {tracing::warn!(%document_id, seq = chunk.seq, error = %e, "抽取调用失败,跳过该分块");