World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
LLM errors preserve their response bodies (#538)
Ben Younes357d4ac45f22
diff --git a/crates/utopia-llm/src/lib.rs b/crates/utopia-llm/src/lib.rsindex 68a1edb4..0151a059 100644--- a/crates/utopia-llm/src/lib.rs+++ b/crates/utopia-llm/src/lib.rs@@ -62,15 +62,15 @@ pub enum ToolStreamItem {}/// **没能从端点拿到一个能解析的回答。** 两种:连不上(DNS、连接、TLS、超时),-/// 或者连上了但回来的根本不是这个 API(响应体解不成 JSON)。+/// 或者连上了但成功响应根本不是这个 API(响应体解不成 JSON)。////// 合成一类是有意的:从用户那边看这两种是同一件事——"你配的这个地址不是模型 API",/// 该做的也是同一件事:去看 URL、看代理。第一版只收传输层失败,结果最常见的那种/// 故障(URL 配错、代理挡在中间回了 HTML)一条告警都不产生,/// 正是"失败无声"本身。///-/// **不含**端点干干净净地回了 4xx/5xx:那说明它就是模型 API,只是密钥、-/// 配额或模型名不对——另一类问题,该找的人也不同。+/// **不含**端点回的 4xx/5xx,即使错误体不是 JSON:那说明服务器已经回答,+/// 错误体本身就是诊断——另一类问题,该找的人也不同。////// 有类型而不是匹配错误文本:调用链上任何一层加一句 context 都会改文本,/// 而 `anyhow` 的 source 链让 `downcast_ref` 一路都认得出来。@@ -147,8 +147,9 @@ fn failure(status: reqwest::StatusCode,retry_after: Option<Duration>,body: &serde_json::Value,+ raw: &str,) -> anyhow::Error {- let detail = err_detail(body);+ let detail = err_detail(body, raw);// **欠费要先判,而且不能只看状态码。**//// 402 是标准答案(SiliconFlow 用它),但 OpenAI 的余额耗尽走的是 **429**,@@ -171,6 +172,17 @@ fn failure(anyhow::anyhow!("{kind} request failed ({status}): {detail}")}+async fn response_failure(+ kind: &str,+ status: reqwest::StatusCode,+ retry_after: Option<Duration>,+ response: reqwest::Response,+) -> anyhow::Result<anyhow::Error> {+ let raw = response.text().await.map_err(Unreachable)?;+ let body = serde_json::from_str(&raw).unwrap_or_default();+ Ok(failure(kind, status, retry_after, &body, &raw))+}+#[derive(Clone)]pub struct LlmClient {http: reqwest::Client,@@ -198,6 +210,9 @@ const CONNECT_TIMEOUT: Duration = Duration::from_secs(10);/// 流水线停摆而**一条错误都不报**(jobs 的孤儿回收只在进程启动时跑一次,/// 进程活着就永远收不了尸)。7459 块的一次灌入死在第 55 块上。const READ_TIMEOUT: Duration = Duration::from_secs(300);+const LLM_REQUEST_KIND: &str = "LLM";+const EMBEDDING_REQUEST_KIND: &str = "Embedding";+const UNKNOWN_ERROR_DETAIL: &str = "unknown error";impl LlmClient {pub fn new(base_url: &str, api_key: Option<&str>, model: &str) -> Self {@@ -248,10 +263,10 @@ impl LlmClient {.map_err(Unreachable)?;let status = resp.status();let retry_after = retry_after_of(resp.headers());- let body: serde_json::Value = resp.json().await.map_err(Unreachable)?;if !status.is_success() {- return Err(failure("LLM", status, retry_after, &body));+ return Err(response_failure(LLM_REQUEST_KIND, status, retry_after, resp).await?);}+ let body: serde_json::Value = resp.json().await.map_err(Unreachable)?;log_usage(&self.model, &body);body["choices"][0]["message"]["content"].as_str()@@ -279,10 +294,10 @@ impl LlmClient {.map_err(Unreachable)?;let status = resp.status();let retry_after = retry_after_of(resp.headers());- let body: serde_json::Value = resp.json().await.map_err(Unreachable)?;if !status.is_success() {- return Err(failure("LLM", status, retry_after, &body));+ return Err(response_failure(LLM_REQUEST_KIND, status, retry_after, resp).await?);}+ let body: serde_json::Value = resp.json().await.map_err(Unreachable)?;let msg = &body["choices"][0]["message"];if msg.is_null() {anyhow::bail!("Unexpected LLM response shape: {body}");@@ -336,8 +351,7 @@ impl LlmClient {if !resp.status().is_success() {let status = resp.status();let retry_after = retry_after_of(resp.headers());- let body: serde_json::Value = resp.json().await.unwrap_or_default();- return Err(failure("LLM", status, retry_after, &body));+ return Err(response_failure(LLM_REQUEST_KIND, status, retry_after, resp).await?);}let mut bytes = resp.bytes_stream();@@ -429,8 +443,7 @@ impl LlmClient {if !resp.status().is_success() {let status = resp.status();let retry_after = retry_after_of(resp.headers());- let body: serde_json::Value = resp.json().await.unwrap_or_default();- return Err(failure("LLM", status, retry_after, &body));+ return Err(response_failure(LLM_REQUEST_KIND, status, retry_after, resp).await?);}let mut bytes = resp.bytes_stream();@@ -474,10 +487,10 @@ impl LlmClient {.map_err(Unreachable)?;let status = resp.status();let retry_after = retry_after_of(resp.headers());- let body: serde_json::Value = resp.json().await.map_err(Unreachable)?;if !status.is_success() {- return Err(failure("Embedding", status, retry_after, &body));+ return Err(response_failure(EMBEDDING_REQUEST_KIND, status, retry_after, resp).await?);}+ let body: serde_json::Value = resp.json().await.map_err(Unreachable)?;let data = body["data"].as_array().ok_or_else(|| anyhow::anyhow!("Unexpected embedding response shape"))?;@@ -519,12 +532,19 @@ fn log_usage(model: &str, body: &serde_json::Value) {);}-fn err_detail(body: &serde_json::Value) -> String {+const MAX_ERROR_DETAIL_CHARS: usize = 500;++fn err_detail(body: &serde_json::Value, raw: &str) -> String {body["error"]["message"].as_str().or_else(|| body["message"].as_str())- .unwrap_or("unknown error")- .to_string()+ .or_else(|| body["error"].as_str())+ .map(String::from)+ .or_else(|| {+ let raw = raw.trim();+ (!raw.is_empty()).then(|| raw.chars().take(MAX_ERROR_DETAIL_CHARS).collect())+ })+ .unwrap_or_else(|| UNKNOWN_ERROR_DETAIL.to_string())}/// 响应体说不说这是余额问题。@@ -545,6 +565,206 @@ fn says_out_of_credit(body: &serde_json::Value) -> bool {#[cfg(test)]mod tests {use super::*;+ use tokio::io::AsyncWriteExt;++ const EPHEMERAL_LOOPBACK: &str = "127.0.0.1:0";+ const TEST_MODEL: &str = "m";+ const TEST_USER_ROLE: &str = "user";+ const TEST_INPUT: &str = "hi";+ const ERROR_STATUS: &str = "502 Bad Gateway";+ const SUCCESS_STATUS: &str = "200 OK";+ const TEXT_CONTENT_TYPE: &str = "text/plain";+ const JSON_CONTENT_TYPE: &str = "application/json";+ const CHAT_SUCCESS_BODY: &str = r#"{"choices":[{"message":{"content":"ok"}}]}"#;+ const EMBEDDING_SUCCESS_BODY: &str = r#"{"data":[{"embedding":[1.0]}]}"#;+ const EMBEDDING_VALUE: f32 = 1.0;+ const SUCCESS_TEXT: &str = "ok";+ const STRING_ERROR: &str = "model is unavailable";+ const NESTED_ERROR: &str = "bad request";+ const NO_RAW_BODY: &str = "";+ const RAW_ERROR_DIAGNOSIS: &str = "proxy says the model context is too long";+ const MULTIBYTE_FIXTURE: &str = "界";+ const INCOMPLETE_BODY_LENGTH: usize = 100;+ const OVER_LIMIT_CHARACTERS: usize = 1;++ fn client_at(addr: std::net::SocketAddr) -> LlmClient {+ LlmClient::new(&format!("http://{addr}"), None, TEST_MODEL)+ }++ fn no_tools() -> serde_json::Value {+ serde_json::json!([])+ }++ async fn an_http_response(+ status: &str,+ content_type: &str,+ body: &str,+ ) -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) {+ let listener = tokio::net::TcpListener::bind(EPHEMERAL_LOOPBACK)+ .await+ .unwrap();+ let addr = listener.local_addr().unwrap();+ let status = status.to_string();+ let content_type = content_type.to_string();+ let body = body.to_string();+ let server = tokio::spawn(async move {+ let (mut socket, _) = listener.accept().await.unwrap();+ let response = format!(+ "HTTP/1.1 {status}\r\nContent-Type: {content_type}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",+ body.len()+ );+ socket.write_all(response.as_bytes()).await.unwrap();+ });+ (addr, server)+ }++ async fn an_http_error(body: &str) -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) {+ an_http_response(ERROR_STATUS, TEXT_CONTENT_TYPE, body).await+ }++ async fn an_incomplete_http_error() -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) {+ let listener = tokio::net::TcpListener::bind(EPHEMERAL_LOOPBACK)+ .await+ .unwrap();+ let addr = listener.local_addr().unwrap();+ let server = tokio::spawn(async move {+ let (mut socket, _) = listener.accept().await.unwrap();+ let response = format!(+ "HTTP/1.1 {ERROR_STATUS}\r\nContent-Length: {INCOMPLETE_BODY_LENGTH}\r\nConnection: close\r\n\r\nx"+ );+ socket.write_all(response.as_bytes()).await.unwrap();+ socket.shutdown().await.unwrap();+ });+ (addr, server)+ }++ async fn assert_diagnosis<T>(+ result: anyhow::Result<T>,+ diagnosis: &str,+ server: tokio::task::JoinHandle<()>,+ ) {+ server.await.unwrap();+ let error = match result {+ Ok(_) => panic!("the endpoint returned an error response"),+ Err(error) => error,+ };+ assert!(!is_unreachable(&error));+ assert!(error.to_string().contains(diagnosis), "{error:#}");+ }++ #[tokio::test]+ async fn a_body_that_is_not_json_is_quoted_and_not_called_unreachable() {+ let diagnosis = RAW_ERROR_DIAGNOSIS;+ let (addr, server) = an_http_error(diagnosis).await;+ let client = client_at(addr);+ assert_diagnosis(+ client+ .chat(&[ChatMessage {+ role: TEST_USER_ROLE.into(),+ content: TEST_INPUT.into(),+ }])+ .await,+ diagnosis,+ server,+ )+ .await;++ let (addr, server) = an_http_error(diagnosis).await;+ let client = client_at(addr);+ assert_diagnosis(client.chat_tools(&[], &no_tools()).await, diagnosis, server).await;++ let (addr, server) = an_http_error(diagnosis).await;+ let client = client_at(addr);+ assert_diagnosis(+ client.chat_tools_stream(&[], &no_tools()).await,+ diagnosis,+ server,+ )+ .await;++ let (addr, server) = an_http_error(diagnosis).await;+ let client = client_at(addr);+ assert_diagnosis(+ client+ .chat_stream(&[ChatMessage {+ role: TEST_USER_ROLE.into(),+ content: TEST_INPUT.into(),+ }])+ .await,+ diagnosis,+ server,+ )+ .await;++ let (addr, server) = an_http_error(diagnosis).await;+ let client = client_at(addr);+ assert_diagnosis(+ client.embed(&[TEST_INPUT.to_string()]).await,+ diagnosis,+ server,+ )+ .await;+ }++ #[tokio::test]+ async fn a_body_that_cannot_be_read_stays_unreachable() {+ let (addr, server) = an_incomplete_http_error().await;+ let client = client_at(addr);+ let result = client.chat(&[]).await;+ server.await.unwrap();+ let error = result.expect_err("the body is incomplete");+ assert!(+ is_unreachable(&error),+ "body read error lost its source: {error:#}"+ );+ }++ #[tokio::test]+ async fn successful_responses_still_parse() {+ let (addr, server) =+ an_http_response(SUCCESS_STATUS, JSON_CONTENT_TYPE, CHAT_SUCCESS_BODY).await;+ let client = client_at(addr);+ let result = client.chat(&[]).await;+ server.await.unwrap();+ assert_eq!(result.unwrap(), SUCCESS_TEXT);++ let (addr, server) =+ an_http_response(SUCCESS_STATUS, JSON_CONTENT_TYPE, CHAT_SUCCESS_BODY).await;+ let client = client_at(addr);+ let result = client.chat_tools(&[], &no_tools()).await;+ server.await.unwrap();+ assert_eq!(result.unwrap().content.as_deref(), Some(SUCCESS_TEXT));++ let (addr, server) =+ an_http_response(SUCCESS_STATUS, JSON_CONTENT_TYPE, EMBEDDING_SUCCESS_BODY).await;+ let client = client_at(addr);+ let result = client.embed(&[TEST_INPUT.to_string()]).await;+ server.await.unwrap();+ assert_eq!(result.unwrap(), vec![vec![EMBEDDING_VALUE]]);+ }++ #[test]+ fn an_error_given_as_a_string_is_forwarded() {+ let body = serde_json::json!({ "error": STRING_ERROR });+ assert_eq!(+ err_detail(&body, NO_RAW_BODY),+ STRING_ERROR,+ "a valid error field must not become unknown"+ );+ }++ #[test]+ fn a_body_with_error_message_still_reads_it() {+ let body = serde_json::json!({ "error": { "message": NESTED_ERROR } });+ assert_eq!(err_detail(&body, NO_RAW_BODY), NESTED_ERROR);+ }++ #[test]+ fn a_long_body_is_truncated() {+ let raw = MULTIBYTE_FIXTURE.repeat(MAX_ERROR_DETAIL_CHARS + OVER_LIMIT_CHARACTERS);+ let detail = err_detail(&serde_json::Value::Null, &raw);+ assert_eq!(detail.chars().count(), MAX_ERROR_DETAIL_CHARS);+ }/// 真发一次注定失败的请求,拿一个货真价实的 `reqwest::Error`。/// 端口 1 上不会有东西监听,而 127.0.0.1 不走代理。@@ -565,7 +785,9 @@ mod tests {/// 请求发得出去,然后没了。连接错误会立刻报,这种不会——没有超时的话/// `send().await` 就永远停在那里,而调用它的 worker 槽再也不释放。async fn a_server_that_never_answers() -> std::net::SocketAddr {- let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();+ let listener = tokio::net::TcpListener::bind(EPHEMERAL_LOOPBACK)+ .await+ .unwrap();let addr = listener.local_addr().unwrap();tokio::spawn(async move {// 收下连接就攥着不放。**必须持有 socket**:一 drop 就是 FIN,@@ -588,7 +810,7 @@ mod tests {let client = LlmClient::with_timeouts(&format!("http://{addr}"),None,- "m",+ TEST_MODEL,Duration::from_secs(5),Duration::from_millis(300),);@@ -596,8 +818,8 @@ mod tests {let out = tokio::time::timeout(std::time::Duration::from_secs(10),client.chat(&[ChatMessage {- role: "user".into(),- content: "hi".into(),+ role: TEST_USER_ROLE.into(),+ content: TEST_INPUT.into(),}]),).await;@@ -692,7 +914,13 @@ mod tests {let body = serde_json::json!({"error": { "message": "You exceeded your current quota", "code": "insufficient_quota" }});- let e = failure("LLM", reqwest::StatusCode::TOO_MANY_REQUESTS, None, &body);+ let e = failure(+ LLM_REQUEST_KIND,+ reqwest::StatusCode::TOO_MANY_REQUESTS,+ None,+ &body,+ NO_RAW_BODY,+ );assert!(out_of_credit(&e).is_some(), "该判成欠费");assert!(rate_limited(&e).is_none(), "不该判成限流");}@@ -701,7 +929,13 @@ mod tests {#[test]fn a_402_is_a_billing_problem() {let body = serde_json::json!({ "message": "Sorry, your account balance is insufficient" });- let e = failure("LLM", reqwest::StatusCode::PAYMENT_REQUIRED, None, &body);+ let e = failure(+ LLM_REQUEST_KIND,+ reqwest::StatusCode::PAYMENT_REQUIRED,+ None,+ &body,+ NO_RAW_BODY,+ );assert!(out_of_credit(&e).is_some());}@@ -709,7 +943,13 @@ mod tests {#[test]fn a_plain_429_is_still_a_rate_limit() {let body = serde_json::json!({ "error": { "message": "TPM limit reached" } });- let e = failure("LLM", reqwest::StatusCode::TOO_MANY_REQUESTS, None, &body);+ let e = failure(+ LLM_REQUEST_KIND,+ reqwest::StatusCode::TOO_MANY_REQUESTS,+ None,+ &body,+ NO_RAW_BODY,+ );assert!(rate_limited(&e).is_some());assert!(out_of_credit(&e).is_none());}