World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
An empty balance is not a rate limit (#182)
WaylandYang4f5bb30713e1
diff --git a/crates/utopia-llm/src/lib.rs b/crates/utopia-llm/src/lib.rsindex 5e182114..68a1edb4 100644--- a/crates/utopia-llm/src/lib.rs+++ b/crates/utopia-llm/src/lib.rs@@ -103,6 +103,26 @@ pub fn rate_limited(err: &anyhow::Error) -> Option<&RateLimited> {err.chain().find_map(|e| e.downcast_ref::<RateLimited>())}+/// 账号付不起这次请求:欠费,或者套餐配额用尽。+///+/// **跟 [`RateLimited`] 分开,因为它不会自己好。** 限流等一分钟就过去,+/// 欠费等到天亮也还是欠费——重试只是在把同一个错误说三遍,而真正该发生的事+/// (有人去充值)不会因为重试而发生。+///+/// 实测:一次跑测里 14 篇文档因为它整篇失败,而当时它跟普通失败走同一条路,+/// 唯一能知道原因的办法是去数据库里翻 `graph_error`。+#[derive(Debug, thiserror::Error)]+#[error("LLM account cannot pay for this request ({status}): {detail}")]+pub struct OutOfCredit {+ pub status: u16,+ pub detail: String,+}++/// anyhow 错误链里的 [`OutOfCredit`],穿透 context 层。+pub fn out_of_credit(err: &anyhow::Error) -> Option<&OutOfCredit> {+ err.chain().find_map(|e| e.downcast_ref::<OutOfCredit>())+}+/// `Retry-After` 的整数秒形态。////// 规范还允许 HTTP-date,这里**不解析**:为一个很少有人发的头引一个日期库不划算,@@ -118,8 +138,10 @@ fn retry_after_of(headers: &reqwest::header::HeaderMap) -> Option<Duration> {.map(Duration::from_secs)}-/// 非 2xx 统一在这里成型。**只有 429 算限流**:503 可能是端点真挂了、-/// 也可能是中间代理,把它算进来会让「等一会儿再来」用在等不回来的地方。+/// 非 2xx 统一在这里成型,分成三类:欠费、限流、其他。+///+/// **503 不算限流**:它可能是端点真挂了、也可能是中间代理,把它算进来会让+/// 「等一会儿再来」用在等不回来的地方。判据窄一点,宁可退回「其他」。fn failure(kind: &str,status: reqwest::StatusCode,@@ -127,6 +149,18 @@ fn failure(body: &serde_json::Value,) -> anyhow::Error {let detail = err_detail(body);+ // **欠费要先判,而且不能只看状态码。**+ //+ // 402 是标准答案(SiliconFlow 用它),但 OpenAI 的余额耗尽走的是 **429**,+ // 靠 body 里的 `insufficient_quota` 区分。只按状态码分类的话,一个没钱的+ // OpenAI 账号会被当成限流,然后无限退避重试一个永远不会好的东西——+ // 而退避越久,症状越像"端点慢",越查不到根上。+ if status == reqwest::StatusCode::PAYMENT_REQUIRED || says_out_of_credit(body) {+ return anyhow::Error::new(OutOfCredit {+ status: status.as_u16(),+ detail,+ });+ }if status == reqwest::StatusCode::TOO_MANY_REQUESTS {return anyhow::Error::new(RateLimited {status: status.as_u16(),@@ -493,6 +527,21 @@ fn err_detail(body: &serde_json::Value) -> String {.to_string()}+/// 响应体说不说这是余额问题。+///+/// **给 429 用的**:OpenAI 用同一个状态码表示「太快了」和「没钱了」,+/// `error.code` 或 `error.type` 里的 `insufficient_quota` 才是分界。+///+/// 只认这一个标识、不去匹配 message 的自由文本:措辞会改、会本地化,+/// 而 `code` 是接口契约的一部分。认不出来就退回按状态码判,那是安全的一侧+/// (当成限流退避几次,比当成欠费直接放弃温和)。+fn says_out_of_credit(body: &serde_json::Value) -> bool {+ ["code", "type"]+ .iter()+ .filter_map(|k| body["error"][k].as_str())+ .any(|v| v == "insufficient_quota")+}+#[cfg(test)]mod tests {use super::*;@@ -635,6 +684,36 @@ mod tests {);}+ /// **429 不一定是限流。** OpenAI 用同一个状态码表示「太快了」和「没钱了」,+ /// 分界在 `error.code`。只按状态码分类的话,一个没钱的账号会被无限退避+ /// 重试——而重试越久,症状越像「端点慢」,越查不到根上。+ #[test]+ fn a_429_that_says_insufficient_quota_is_a_billing_problem() {+ let body = serde_json::json!({+ "error": { "message": "You exceeded your current quota", "code": "insufficient_quota" }+ });+ let e = failure("LLM", reqwest::StatusCode::TOO_MANY_REQUESTS, None, &body);+ assert!(out_of_credit(&e).is_some(), "该判成欠费");+ assert!(rate_limited(&e).is_none(), "不该判成限流");+ }++ /// 402 是标准答案,SiliconFlow 用的就是它。+ #[test]+ fn a_402_is_a_billing_problem() {+ let body = serde_json::json!({ "message": "Sorry, your account balance is insufficient" });+ let e = failure("LLM", reqwest::StatusCode::PAYMENT_REQUIRED, None, &body);+ assert!(out_of_credit(&e).is_some());+ }++ /// 反面:不带那个标识的 429 还是限流,别把会自己好的事判成要人动手。+ #[test]+ fn a_plain_429_is_still_a_rate_limit() {+ let body = serde_json::json!({ "error": { "message": "TPM limit reached" } });+ let e = failure("LLM", reqwest::StatusCode::TOO_MANY_REQUESTS, None, &body);+ assert!(rate_limited(&e).is_some());+ assert!(out_of_credit(&e).is_none());+ }+/// 反面:别的 4xx 不是限流。密钥错了重试一万次还是错。#[tokio::test]async fn an_auth_failure_is_not_a_rate_limit() {diff --git a/crates/utopia-server/src/alerting.rs b/crates/utopia-server/src/alerting.rsindex 8b83069d..89f36fdb 100644--- a/crates/utopia-server/src/alerting.rs+++ b/crates/utopia-server/src/alerting.rs@@ -56,12 +56,19 @@ pub async fn observe_job_failure(/// 一次任务失败该报哪一类告警,报不报。////// **抽成纯函数是为了能测**:`observe_job_failure` 要 `AppState` 和一个真数据库,-/// 而这里要守的是分类本身——限流不能被当成端点不可达,反之亦然。两者该做的事-/// 完全不同:一个去查网络或地址,一个去降并发或升配额。+/// 而这里要守的是分类本身。三类该做的事完全不同——欠费要人去充值,限流要降并发+/// 或升配额,端点不可达要查网络和地址。**混成一条会把人指向错的地方,比不报还费时间。**///-/// **限流排在前面。** 两者今天互斥,但顺序反过来的话,将来谁给 `Unreachable`-/// 加一层包装就会把限流吞进去,而症状是这条告警再也不出现——没有任何测试会红。+/// 顺序是「更该先说的放前面」,而不是判据有重叠:类型本身互斥,重叠在状态码上+/// (OpenAI 的余额耗尽也回 429),那一层已经在 `utopia_llm::failure` 里分开了。fn alert_for(err: &anyhow::Error) -> Option<(&'static str, &'static str)> {+ // 欠费排在限流之前。两者的状态码会重叠——OpenAI 的余额耗尽也回 429——+ // 而分类在 `utopia_llm::failure` 里已经做完,这里的顺序只是把「更该先说+ // 的那件事」放前面:没钱是人必须动手的,限流不是+ if utopia_llm::out_of_credit(err).is_some() {+ // `error`:它不会自己好,在有人充值之前抽取一直是停的+ return Some((alerts::kind::LLM_OUT_OF_CREDIT, "error"));+ }if utopia_llm::rate_limited(err).is_some() {// `warning` 不是 `error`:配额会自己恢复,端点挂了不会return Some((alerts::kind::LLM_RATE_LIMITED, "warning"));@@ -135,7 +142,7 @@ pub async fn observe_schema_sync_failure(#[cfg(test)]mod tests {use super::alert_for;- use utopia_llm::RateLimited;+ use utopia_llm::{OutOfCredit, RateLimited};use utopia_store::alerts::kind;/// 限流与端点不可达是两类事,报的告警必须分开。@@ -167,6 +174,30 @@ mod tests {assert_eq!(alert_for(&err), Some((kind::LLM_RATE_LIMITED, "warning")));}+ /// 欠费与限流必须分开:一个要人去充值,一个等一会儿就好。+ ///+ /// 报错了方向的代价是实打实的——实测 14 篇文档因为欠费整篇失败,+ /// 而当时唯一能知道原因的办法是去数据库里翻 `graph_error`。+ #[test]+ fn out_of_credit_is_not_a_rate_limit() {+ let err = anyhow::Error::new(OutOfCredit {+ status: 402,+ detail: "Sorry, your account balance is insufficient".into(),+ });+ assert_eq!(alert_for(&err), Some((kind::LLM_OUT_OF_CREDIT, "error")));+ }++ /// 判定同样要穿透 context 层。+ #[test]+ fn out_of_credit_survives_context_layers() {+ let err = anyhow::Error::new(OutOfCredit {+ status: 402,+ detail: "insufficient balance".into(),+ })+ .context("bootstrap_ontology 失败");+ assert_eq!(alert_for(&err), Some((kind::LLM_OUT_OF_CREDIT, "error")));+ }+/// 反面:普通失败不该报告警,否则每一次解析错都弹一条,人会学会忽略它。#[test]fn an_ordinary_failure_raises_nothing() {diff --git a/crates/utopia-store/src/alerts.rs b/crates/utopia-store/src/alerts.rsindex f5e9d91c..bc67a75e 100644--- a/crates/utopia-store/src/alerts.rs+++ b/crates/utopia-store/src/alerts.rs@@ -39,6 +39,12 @@ pub mod kind {/// 真的被配额挡在外面时,没有这条告警就没有任何人知道——/// 实测一次跑测里 4 篇失败,一半是不可达(有告警),一半是限流(静默)。pub const LLM_RATE_LIMITED: &str = "llm.rate_limited";+ /// 系统级:账号付不起请求——欠费或套餐配额用尽。`min_role = admin`+ ///+ /// **`error` 而不是 `warning`,正因为它跟限流的区别是「会不会自己好」**:+ /// 配额到点重置,欠费不会。在有人去充值之前,这个部署的抽取与向量化+ /// 一直是停的。+ pub const LLM_OUT_OF_CREDIT: &str = "llm.out_of_credit";/// 库级:数据源挂上了,它的库表结构却没摄进来。`min_role = admin`////// **这一条描述的不是那次失败,是它留下的状态**:源挂着,而问数看不见它有diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.tsindex 51868268..a133c070 100644--- a/web/src/i18n/en.ts+++ b/web/src/i18n/en.ts@@ -168,6 +168,10 @@ export const en = {title: "The model endpoint is rate limiting us",hint: "Documents were retried and still turned away, so some are missing facts. Lower model concurrency in system settings, or raise the quota on the account.",},+ "llm.out_of_credit": {+ title: "The model account cannot pay for requests",+ hint: "Extraction and embedding are stopped and will not resume on their own. Top up the account, or point system settings at an endpoint that can serve.",+ },} as Record<string, { title: string; hint: string } | undefined>,// 没见过的 kind 也要能显示:新告警源上线时前端可能还没更新unknownKind: (kind: string) => kind,diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.tsindex 121a8518..8ee98724 100644--- a/web/src/i18n/zh.ts+++ b/web/src/i18n/zh.ts@@ -149,6 +149,10 @@ export const zh: Strings = {title: "模型端点在限流",hint: "退避重试之后仍然被挡回来,有文档因此缺了事实。去系统设置里降低模型并发,或者把账号配额升一档。",},+ "llm.out_of_credit": {+ title: "模型账号付不起请求",+ hint: "抽取与向量化已停摆,而且不会自己恢复。去给账号充值,或者在系统设置里换一个能用的端点。",+ },} as Record<string, { title: string; hint: string } | undefined>,unknownKind: (kind: string) => kind,},