World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Being turned away by a quota is worth telling someone about (#176)
WaylandYang0f0bd6850422
diff --git a/crates/utopia-server/src/alerting.rs b/crates/utopia-server/src/alerting.rsindex 0f12ed64..8b83069d 100644--- a/crates/utopia-server/src/alerting.rs+++ b/crates/utopia-server/src/alerting.rs@@ -19,8 +19,8 @@ const RETAIN_DAYS: i32 = 30;/// 就是同一件事发三遍——重试本来就是为了不惊动人,报出去等于把它的意义抵消掉。/// 一个任务至多一条告警,不需要任何去重状态。///-/// 判据是错误链里有没有 [`utopia_llm::Unreachable`],**不匹配错误文本**——-/// 调用链上任何一层加一句 context 都会改文本。+/// 判据是错误的**类型**,不匹配错误文本——调用链上任何一层加一句 context+/// 都会改文本。分类本身抽在 [`alert_for`]。pub async fn observe_job_failure(state: &AppState,job: &utopia_store::jobs::Job,@@ -29,16 +29,16 @@ pub async fn observe_job_failure(if job.attempts < job.max_attempts {return;}- if !utopia_llm::is_unreachable(err) {+ let Some((kind, severity)) = alert_for(err) else {return;- }- // 系统级:哪个库的任务撞上的不重要,端点是整个部署共用的+ };+ // 系统级:哪个库的任务撞上的不重要,端点与配额是整个部署共用的if let Err(e) = alerts::raise(&state.pool,alerts::NewAlert {kb_id: None,- severity: "error",- kind: alerts::kind::LLM_UNREACHABLE,+ severity,+ kind,min_role: Role::Admin,subject_type: Some("system"),subject_id: None,@@ -47,12 +47,31 @@ pub async fn observe_job_failure().await{- tracing::warn!(error = %e, "上报 llm.unreachable 失败");+ tracing::warn!(error = %e, %kind, "上报告警失败");return;}state.emit_alert();}+/// 一次任务失败该报哪一类告警,报不报。+///+/// **抽成纯函数是为了能测**:`observe_job_failure` 要 `AppState` 和一个真数据库,+/// 而这里要守的是分类本身——限流不能被当成端点不可达,反之亦然。两者该做的事+/// 完全不同:一个去查网络或地址,一个去降并发或升配额。+///+/// **限流排在前面。** 两者今天互斥,但顺序反过来的话,将来谁给 `Unreachable`+/// 加一层包装就会把限流吞进去,而症状是这条告警再也不出现——没有任何测试会红。+fn alert_for(err: &anyhow::Error) -> Option<(&'static str, &'static str)> {+ if utopia_llm::rate_limited(err).is_some() {+ // `warning` 不是 `error`:配额会自己恢复,端点挂了不会+ return Some((alerts::kind::LLM_RATE_LIMITED, "warning"));+ }+ if utopia_llm::is_unreachable(err) {+ return Some((alerts::kind::LLM_UNREACHABLE, "error"));+ }+ None+}+/// 每天清一次过期告警。pub fn spawn_retention_sweep(state: AppState) {tokio::spawn(async move {@@ -112,3 +131,54 @@ pub async fn observe_schema_sync_failure(}state.emit_alert();}++#[cfg(test)]+mod tests {+ use super::alert_for;+ use utopia_llm::RateLimited;+ use utopia_store::alerts::kind;++ /// 限流与端点不可达是两类事,报的告警必须分开。+ ///+ /// 混成一条的后果不是"标签不好看":`llm.unreachable` 说的是端点连不上,+ /// 管理员会去查网络和地址,而真正该做的是降并发或者升配额——**告警把人+ /// 指向了错误的地方**,比不报还费时间。+ #[test]+ fn a_rate_limit_is_not_an_unreachable_endpoint() {+ let err = anyhow::Error::new(RateLimited {+ status: 429,+ retry_after: None,+ detail: "TPM limit reached".into(),+ });+ assert_eq!(alert_for(&err), Some((kind::LLM_RATE_LIMITED, "warning")));+ }++ /// **判定要穿透 context 层。** 抽取那条路上至少加了一句+ /// 「限流退避 5 次仍未通过」,靠文本匹配的判定当天就废。+ #[test]+ fn it_survives_context_layers() {+ let err = anyhow::Error::new(RateLimited {+ status: 429,+ retry_after: None,+ detail: "slow down".into(),+ })+ .context("限流退避 5 次仍未通过")+ .context("extract_document 失败");+ assert_eq!(alert_for(&err), Some((kind::LLM_RATE_LIMITED, "warning")));+ }++ /// 反面:普通失败不该报告警,否则每一次解析错都弹一条,人会学会忽略它。+ #[test]+ fn an_ordinary_failure_raises_nothing() {+ let err = anyhow::anyhow!("结果解析失败").context("抽取失败");+ assert_eq!(alert_for(&err), None);+ }++ /// 反面:干净的 4xx 不是限流也不是不可达。密钥错了要换密钥,+ /// 而那既不该降并发也不该查网络。+ #[test]+ fn an_auth_failure_raises_nothing() {+ let err = anyhow::anyhow!("LLM request failed (401 Unauthorized): bad key");+ assert_eq!(alert_for(&err), None);+ }+}diff --git a/crates/utopia-store/src/alerts.rs b/crates/utopia-store/src/alerts.rsindex a500c01c..f5e9d91c 100644--- a/crates/utopia-store/src/alerts.rs+++ b/crates/utopia-store/src/alerts.rs@@ -25,8 +25,20 @@ pub mod kind {/// 库级:某个来源同步失败。`min_role = editor`pub const SOURCE_SYNC_FAILED: &str = "source.sync_failed";/// 系统级:模型端点没给出可用的回答——连不上,或者连上了但回来的不是这个 API。- /// 端点干净地回 4xx 不算:那说明它就是模型 API,只是密钥或配额不对+ /// 端点干净地回 4xx 不算:那说明它就是模型 API,只是密钥或配额不对。+ /// 配额那一种见 [`LLM_RATE_LIMITED`]pub const LLM_UNREACHABLE: &str = "llm.unreachable";+ /// 系统级:端点在限流,退避重试用尽后仍然过不去。`min_role = admin`+ ///+ /// **与 [`LLM_UNREACHABLE`] 分开,因为该做的事不同**:端点不可达要去查+ /// 网络或地址,配额打满要去降并发或升档,找的人和动作都不一样。+ ///+ /// severity 是 `warning` 不是 `error`:配额会自己恢复,端点挂了不会。+ ///+ /// 它补的是「退避重试」留下的那一半。重试之后不再丢数据,但一篇文档+ /// 真的被配额挡在外面时,没有这条告警就没有任何人知道——+ /// 实测一次跑测里 4 篇失败,一半是不可达(有告警),一半是限流(静默)。+ pub const LLM_RATE_LIMITED: &str = "llm.rate_limited";/// 库级:数据源挂上了,它的库表结构却没摄进来。`min_role = admin`////// **这一条描述的不是那次失败,是它留下的状态**:源挂着,而问数看不见它有diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.tsindex 8c05a764..d27da455 100644--- a/web/src/i18n/en.ts+++ b/web/src/i18n/en.ts@@ -158,6 +158,10 @@ export const en = {title: "The model endpoint gave no usable answer",hint: "Extraction and embedding are stopped. Check the endpoint URL in system settings.",},+ "llm.rate_limited": {+ title: "The model endpoint is rate limiting us",+ hint: "Documents were retried and still turned away, so some are missing facts. Lower model concurrency in system settings, or raise the quota on the account.",+ },} as Record<string, { title: string; hint: string } | undefined>,// 没见过的 kind 也要能显示:新告警源上线时前端可能还没更新unknownKind: (kind: string) => kind,diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.tsindex f1c058c4..28ba6481 100644--- a/web/src/i18n/zh.ts+++ b/web/src/i18n/zh.ts@@ -140,6 +140,10 @@ export const zh: Strings = {title: "模型端点没有给出可用的回答",hint: "抽取与向量化已停摆。去系统设置里检查端点地址。",},+ "llm.rate_limited": {+ title: "模型端点在限流",+ hint: "退避重试之后仍然被挡回来,有文档因此缺了事实。去系统设置里降低模型并发,或者把账号配额升一档。",+ },} as Record<string, { title: string; hint: string } | undefined>,unknownKind: (kind: string) => kind,},