World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
The token ceiling is ours, and a cut reply says so (#786)
WaylandYang4960e788fe27
diff --git a/crates/utopia-llm/src/lib.rs b/crates/utopia-llm/src/lib.rsindex d5d60892..703c509d 100644--- a/crates/utopia-llm/src/lib.rs+++ b/crates/utopia-llm/src/lib.rs@@ -290,6 +290,26 @@ async fn response_failure(Ok(failure(kind, status, retry_after, &body, &raw))}+/// 流式问答的结果:整段答案,连端点给的收尾原因。+///+/// **为什么原因要带出来**:`finish_reason: length` 是端点说「我是被截断的」,+/// 而截断的回复长得和完整的一模一样——解析器只看得见 JSON 少了尾巴,看不见+/// 少的原因。带着它,丢弃行才能说「撞上了 token 上限」,而不只是「解析不了」。+#[derive(Debug)]+pub struct Reply {+ pub text: String,+ /// 端点给的收尾原因(`stop` / `length` / …)。流里没有这一项就是 `None`:+ /// 有的实现只发 `[DONE]`,缺席不代表答案是完整的+ pub finish_reason: Option<String>,+}++impl Reply {+ /// 端点说这段是被 token 上限截断的。+ pub fn hit_token_ceiling(&self) -> bool {+ self.finish_reason.as_deref() == Some("length")+ }+}+#[derive(Clone)]pub struct LlmClient {http: reqwest::Client,@@ -318,6 +338,27 @@ const CONNECT_TIMEOUT: Duration = Duration::from_secs(10);/// 进程活着就永远收不了尸)。7459 块的一次灌入死在第 55 块上。const READ_TIMEOUT: Duration = Duration::from_secs(300);+/// **一次回复我们要多少 token**(#760)。+///+/// 不送这个字段,答案在哪里断由端点自己的默认值说了算:关掉推理时,一块密集+/// 正文的答案实测停在**正好 4,096** 个 completion token 上,`finish_reason`+/// 是 `length`,那段 JSON 于是解析不了。上限归我们,答案才在我们定的地方断。+///+/// 为什么是 65,536,而不是照着答案的长度定:`max_tokens` 在各家不是一个意思。+/// Anthropic 把思考算在里面,OpenAI 的 `max_completion_tokens` 也把推理 token+/// 算进去。在那样的端点上,这个数就不是答案的天花板,而是**思考的预算**——而+/// 0044 的「What a reasoning cap costs」量过把思考压到一万五六:885 条陈述掉到+/// 729,表格数字漏掉从 2% 涨到 16%。那正是被否掉的那个杆,不能让它从这里溜回来。+///+/// 这条路上量到的最大一次完成是 45,428 个 token(开着推理)。取它之上的一档,+/// 于是它永远只是「防端点那 4,096 的默认值」,永远变不成推理上限。答案本身约+/// 1,500 个 token,离这条线远得很。+///+/// **开着推理时这个端点不读它**——一次 capped 到 4,096 的调用回了 45,428 个+/// completion token 还正常收尾。送它因此是「读的地方有用,不读的地方不亏」,+/// 不是一条能指望所有端点都认的保证。+const MAX_COMPLETION_TOKENS: u32 = 65_536;+/// 推理模型的思考过程不该进下游(#690)。////// DeepSeek-V4-Flash 这类推理模型把思考过程以内联 `<think>…</think>` 写进@@ -429,7 +470,7 @@ impl LlmClient {&self,messages: &[ChatMessage],temperature: Option<f32>,- ) -> anyhow::Result<String> {+ ) -> anyhow::Result<Reply> {let mut body = json!({"model": self.model,"messages": messages,@@ -437,6 +478,8 @@ impl LlmClient {// 用量随最后一帧回来:比较两次运行的第一件事是看 completion token,// 换成流式不能把这个数弄丢"stream_options": { "include_usage": true },+ // 上限归我们,不归端点的默认值([`MAX_COMPLETION_TOKENS`])+ "max_tokens": MAX_COMPLETION_TOKENS,});if let Some(t) = temperature {body["temperature"] = json!(t);@@ -455,6 +498,7 @@ impl LlmClient {let mut bytes = resp.bytes_stream();let (mut buf, mut answer) = (Vec::new(), String::new());let (mut saw_frame, mut ended) = (false, false);+ let mut finish_reason: Option<String> = None;while let Some(part) = bytes.next().await {let part = part.map_err(Unreachable)?;// 网络片段可能断在 UTF-8 字符中间,等完整 SSE 帧到齐再解码。@@ -463,13 +507,25 @@ impl LlmClient {while let Some(pos) = buf.windows(2).position(|w| w == b"\n\n") {let frame = String::from_utf8_lossy(&buf[..pos]).into_owned();buf.drain(..pos + 2);- self.take_frame(&frame, &mut answer, &mut saw_frame, &mut ended);+ self.take_frame(+ &frame,+ &mut answer,+ &mut saw_frame,+ &mut ended,+ &mut finish_reason,+ );}}// **收尾那一帧也算**:末尾不跟空行的实现有的是,丢掉它就是丢掉答案的尾巴let rest = String::from_utf8_lossy(&buf);if !rest.trim().is_empty() {- self.take_frame(&rest, &mut answer, &mut saw_frame, &mut ended);+ self.take_frame(+ &rest,+ &mut answer,+ &mut saw_frame,+ &mut ended,+ &mut finish_reason,+ );}if !saw_frame {anyhow::bail!("LLM stream carried no frames");@@ -481,11 +537,21 @@ impl LlmClient {got: answer.chars().count(),}));}- Ok(strip_reasoning(&answer).to_string())+ Ok(Reply {+ text: strip_reasoning(&answer).to_string(),+ finish_reason,+ })}/// 一个 SSE 帧:取内容增量、认终止信号、顺手记用量。推理的增量读都不读。- fn take_frame(&self, frame: &str, answer: &mut String, saw_frame: &mut bool, ended: &mut bool) {+ fn take_frame(+ &self,+ frame: &str,+ answer: &mut String,+ saw_frame: &mut bool,+ ended: &mut bool,+ finish_reason: &mut Option<String>,+ ) {for line in frame.lines() {let Some(data) = line.strip_prefix("data:").map(str::trim) else {continue;@@ -502,9 +568,13 @@ impl LlmClient {answer.push_str(delta);}// 模型自己说完了:正常收尾(stop)或撞上它的输出上限(length),两种都是- // 端点把话说完了,与「流断在半路」不同- if v["choices"][0]["finish_reason"].is_string() {+ // 端点把话说完了,与「流断在半路」不同。+ //+ // 是哪一种要留下来:两种都让 `ended` 为真,可 `length` 的回复是半截的,+ // 不带出去就只剩「解析不了」,说不出它为什么不全(#760)+ if let Some(reason) = v["choices"][0]["finish_reason"].as_str() {*ended = true;+ *finish_reason = Some(reason.to_string());}// 用量只在最后一帧(choices 为空)出现if !v["usage"].is_null() {@@ -912,14 +982,14 @@ mod tests {/// **必须先读再答。** 收到的数据还没读就关连接,Windows 会发 RST,客户端那边/// 已经到手的响应连同错误一起变成「连接被中止」(os error 10053)——这组测试/// 在 Linux 上绿、在 Windows 上红,就是这个原因- async fn read_request(socket: &mut tokio::net::TcpStream) {+ async fn read_request(socket: &mut tokio::net::TcpStream) -> String {use tokio::io::AsyncReadExt;let mut buf = Vec::new();let mut chunk = [0u8; 1024];let header_end = loop {let n = socket.read(&mut chunk).await.unwrap();if n == 0 {- return;+ return String::from_utf8_lossy(&buf).into_owned();}buf.extend_from_slice(&chunk[..n]);if let Some(pos) = buf.windows(4).position(|w| w == b"\r\n\r\n") {@@ -939,6 +1009,7 @@ mod tests {}buf.extend_from_slice(&chunk[..n]);}+ String::from_utf8_lossy(&buf).into_owned()}/// 一个只答一次的 HTTP 服务:读完请求,把这份响应原样写回去,关掉。@@ -956,7 +1027,7 @@ mod tests {let body = body.to_string();let server = tokio::spawn(async move {let (mut socket, _) = listener.accept().await.unwrap();- read_request(&mut socket).await;+ let _ = read_request(&mut socket).await;let response = format!("HTTP/1.1 {status}\r\nContent-Type: {content_type}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",body.len()@@ -967,6 +1038,37 @@ mod tests {(addr, server)}+ /// 同上,另外把**请求**原样交回来:要断言的是我们发出去了什么+ /// (`max_tokens` 有没有真的写进请求体),不是端点答了什么。+ async fn an_http_response_capturing(+ status: &str,+ content_type: &str,+ body: &str,+ ) -> (+ std::net::SocketAddr,+ tokio::task::JoinHandle<()>,+ tokio::sync::oneshot::Receiver<String>,+ ) {+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();+ let addr = listener.local_addr().unwrap();+ let (tx, rx) = tokio::sync::oneshot::channel();+ let status = status.to_string();+ let content_type = content_type.to_string();+ let body = body.to_string();+ let server = tokio::spawn(async move {+ let (mut socket, _) = listener.accept().await.unwrap();+ let request = read_request(&mut socket).await;+ let _ = tx.send(request);+ let response = format!(+ "HTTP/1.1 {status}\r\nContent-Type: {content_type}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",+ body.len()+ );+ socket.write_all(response.as_bytes()).await.unwrap();+ socket.shutdown().await.unwrap();+ });+ (addr, server, rx)+ }+// HTTP 分块可以断在 UTF-8 字符中间,与 SSE 帧边界无关。async fn bytewise_sse(body: &str) -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) {let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();@@ -1004,9 +1106,10 @@ mod tests {#[tokio::test]async fn bytewise_utf8_survives_collected_streaming() {let (addr, server) = bytewise_sse(&unicode_sse()).await;+ // 这一刀之后整段回复带着 finish_reason 一起回来,正文在 `text` 上let answer = client_at(addr).chat_at_streaming(&[], None).await.unwrap();server.await.unwrap();- assert_eq!(answer, "你好🦀");+ assert_eq!(answer.text, "你好🦀");}#[tokio::test]@@ -1035,7 +1138,6 @@ mod tests {assert_eq!(delta, "你好🦀");assert_eq!(turn.content.as_deref(), Some("你好🦀"));assert_eq!(turn.tool_calls.len(), 1);- assert_eq!(turn.tool_calls[0].arguments, "{\"city\":\"杭州\"}");}async fn an_http_error(body: &str) -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) {@@ -1048,7 +1150,7 @@ mod tests {let addr = listener.local_addr().unwrap();let server = tokio::spawn(async move {let (mut socket, _) = listener.accept().await.unwrap();- read_request(&mut socket).await;+ let _ = read_request(&mut socket).await;socket.write_all(b"HTTP/1.1 502 Bad Gateway\r\nContent-Length: 100\r\nConnection: close\r\n\r\nx").await@@ -1209,7 +1311,7 @@ mod tests {.await.unwrap();server.await.unwrap();- assert_eq!(answer, "{\"e\":[]}", "只收 content 的增量,拼成整段");+ assert_eq!(answer.text, "{\"e\":[]}", "只收 content 的增量,拼成整段");// 开了流却一帧都没发:那不是空答案,那是没答// 末帧不跟空行:尾巴不能丢@@ -1228,7 +1330,12 @@ data: {\"choices\":[{\"delta\":{\"content\":\"tail\"},\"finish_reason\":\"stop\".await.unwrap();server.await.unwrap();- assert_eq!(answer, "headtail", "最后一帧没有空行收尾,也要算进去");+ assert_eq!(answer.text, "headtail", "最后一帧没有空行收尾,也要算进去");+ assert_eq!(+ answer.finish_reason.as_deref(),+ Some("stop"),+ "端点给的收尾原因要带出来"+ );// 开口了又半路没了:那不是一个短答案,那是没答完,值得再试一次let cut = "data: {\"choices\":[{\"delta\":{\"content\":\"half an ans\"}}]}@@ -1267,6 +1374,77 @@ data: {\"choices\":[{\"delta\":{\"content\":\"tail\"},\"finish_reason\":\"stop\"assert!(format!("{err:#}").contains("no frames"), "{err:#}");}+ /// 上限归我们,被截断这件事说得出来(#760)。+ ///+ /// 两件事一起测,因为它们是同一个毛病的两半:不送 `max_tokens`,答案在哪里+ /// 断由端点的默认值说了算;送了却把 `finish_reason` 丢掉,断了也没人知道为什么。+ #[tokio::test]+ async fn a_token_ceiling_is_ours_and_a_cut_reply_says_so() {+ // `length` = 端点说「我是撞上上限停的」。这样的回复长得和正常收尾一模一样+ let cut = [+ r#"data: {"choices":[{"delta":{"content":"{\"e\":[{\"n\":\"half"}}]}"#,+ r#"data: {"choices":[{"delta":{},"finish_reason":"length"}]}"#,+ "data: [DONE]",+ "",+ ]+ .join(+ "++",+ );+ let (addr, server, request) =+ an_http_response_capturing("200 OK", "text/event-stream", &cut).await;+ let reply = client_at(addr)+ .chat_at_streaming(+ &[ChatMessage {+ role: "user".into(),+ content: "hi".into(),+ }],+ Some(0.0),+ )+ .await+ .unwrap();+ server.await.unwrap();++ // 一、请求里真的带了上限,而且是我们那个数+ let sent = request.await.unwrap();+ let body: serde_json::Value =+ serde_json::from_str(sent.split_once("\r\n\r\n").expect("请求该有 body").1)+ .expect("请求体是 JSON");+ assert_eq!(+ body["max_tokens"],+ json!(MAX_COMPLETION_TOKENS),+ "不送这个字段,答案断在哪里就是端点的默认值说了算:{body}"+ );++ // 二、被截断这件事到得了调用方。答案照旧是完整的那半段——半截的回复+ // 仍然有价值,丢的是尾巴不是全部+ assert!(reply.hit_token_ceiling(), "{:?}", reply.finish_reason);+ assert_eq!(reply.finish_reason.as_deref(), Some("length"));+ assert!(reply.text.starts_with(r#"{"e":["#), "{}", reply.text);++ // 正常收尾不是截断:两者都让流正常结束,只有原因分得开+ let whole =+ "data: {\"choices\":[{\"delta\":{\"content\":\"{}\"},\"finish_reason\":\"stop\"}]}++data: [DONE]++";+ let (addr, server) = an_http_response("200 OK", "text/event-stream", whole).await;+ let reply = client_at(addr)+ .chat_at_streaming(+ &[ChatMessage {+ role: "user".into(),+ content: "hi".into(),+ }],+ None,+ )+ .await+ .unwrap();+ server.await.unwrap();+ assert!(!reply.hit_token_ceiling(), "stop 不是截断");+ }+/// 会自己好的与不会自己好的分开:网关那几个是 [`Unavailable`],密钥错那类照旧/// 是 [`Rejected`],欠费与限流各归各位。调用方据此决定「等一会儿再来」还是「这块废了」#[tokio::test]diff --git a/crates/utopia-server/src/extraction.rs b/crates/utopia-server/src/extraction.rsindex 68f22bd0..9d5a1a4f 100644--- a/crates/utopia-server/src/extraction.rs+++ b/crates/utopia-server/src/extraction.rs@@ -52,7 +52,7 @@ pub(crate) async fn chat_retrying_rate_limits_at(client: &utopia_llm::LlmClient,messages: &[utopia_llm::ChatMessage],temperature: Option<f32>,-) -> anyhow::Result<String> {+) -> anyhow::Result<utopia_llm::Reply> {let mut backoff = Duration::from_secs(2);for attempt in 1..=RATE_LIMIT_TRIES {// 许可只包住调用本身,出了这个块就还回去。diff --git a/crates/utopia-server/src/extraction_open.rs b/crates/utopia-server/src/extraction_open.rsindex 1bb4e658..fac83eb1 100644--- a/crates/utopia-server/src/extraction_open.rs+++ b/crates/utopia-server/src/extraction_open.rs@@ -242,18 +242,34 @@ pub(crate) async fn run_open(continue;}};- tracing::debug!(%document_id, seq = chunk.seq, reply = %reply, "开放抽取的原始回复");- let extraction = match utopia_extract::open::parse_open_response(&reply) {+ tracing::debug!(%document_id, seq = chunk.seq, reply = %reply.text, "开放抽取的原始回复");+ // 端点说它是撞上 token 上限停的。解析器只看得见 JSON 少了尾巴,看不见+ // 少的原因,所以这句话得从回复里带过来(#760)+ let cut_by_ceiling = reply.hit_token_ceiling();+ let extraction = match utopia_extract::open::parse_open_response(&reply.text) {Ok(x) => x,Err(e) => {- tracing::warn!(%document_id, seq = chunk.seq, error = %e, "开放抽取回复解析失败,跳过该分块");+ tracing::warn!(%document_id, seq = chunk.seq, error = %e, hit_token_ceiling = cut_by_ceiling, "开放抽取回复解析失败,跳过该分块");+ // 一整块没进图,而原因不同:撞上上限是「答案太长」,改得动;+ // 别的解析失败是「回复不合结构」。两种都写成同一句话就分不开了+ let (why, detail) = if cut_by_ceiling {+ (+ "回复撞上 token 上限被截断,剩下的解析不了,这一块没有进图",+ format!("#{}:hit the token ceiling;{e}", chunk.seq),+ )+ } else {+ (+ "回复解析不了,这一块没有进图",+ format!("#{}:{e}", chunk.seq),+ )+ };drop_signal(state,kb_id,document_id,reason::CHUNK_UNEXTRACTED,- "回复解析不了,这一块没有进图",- Some(&format!("#{}:{e}", chunk.seq)),+ why,+ Some(&detail),).await;unextracted.push((chunk.seq, format!("回复解析失败:{e}")));@@ -266,7 +282,11 @@ pub(crate) async fn run_open(kb_id,document_id,reason::TRUNCATED_REPLY,- "the open reply was cut off; kept up to the last complete item",+ if cut_by_ceiling {+ "the open reply hit the token ceiling; kept up to the last complete item"+ } else {+ "the open reply was cut off; kept up to the last complete item"+ },None,).await;diff --git a/crates/utopia-server/src/phrase_alignment.rs b/crates/utopia-server/src/phrase_alignment.rsindex 827a7f32..e81ece69 100644--- a/crates/utopia-server/src/phrase_alignment.rs+++ b/crates/utopia-server/src/phrase_alignment.rs@@ -202,7 +202,7 @@ async fn align_phrases_locked(continue;}};- let (choices, malformed) = match parse_phrase_response(&reply, &items) {+ let (choices, malformed) = match parse_phrase_response(&reply.text, &items) {Ok(x) => x,Err(e) => {tracing::warn!(%kb_id, error = %e, "短语对齐回复解析失败,这一批留到下次");diff --git a/crates/utopia-server/src/time_resolution.rs b/crates/utopia-server/src/time_resolution.rsindex bb09b247..dc1a5c11 100644--- a/crates/utopia-server/src/time_resolution.rs+++ b/crates/utopia-server/src/time_resolution.rs@@ -382,7 +382,7 @@ pub async fn resolve_document(state: &AppState, document_id: Uuid) -> anyhow::Relet reply =chat_retrying_rate_limits_at(state, &settings, &client, &messages, Some(0.0)).await?;- let mut dating = parse_dating_response(&reply)?;+ let mut dating = parse_dating_response(&reply.text)?;// 日期词要在开头里核对得到(与名字、引文同一条规矩);核不到的日期不算match dating.date_words.as_deref() {Some(words) if span_in_quote(words, &opening) => {}@@ -449,7 +449,7 @@ pub async fn resolve_document(state: &AppState, document_id: Uuid) -> anyhow::Recontinue;}};- let (interps, skipped) = match parse_interpretation_response(&reply, &ids) {+ let (interps, skipped) = match parse_interpretation_response(&reply.text, &ids) {Ok(x) => x,Err(e) => {tracing::warn!(%document_id, error = %e, "时间解释回复解析失败,这一批留作未解析");diff --git a/crates/utopia-server/src/type_alignment.rs b/crates/utopia-server/src/type_alignment.rsindex 0d0f4a57..703f2b63 100644--- a/crates/utopia-server/src/type_alignment.rs+++ b/crates/utopia-server/src/type_alignment.rs@@ -209,7 +209,7 @@ async fn align_types_locked(continue;}};- let (choices, malformed) = match parse_kind_word_response(&reply, &items) {+ let (choices, malformed) = match parse_kind_word_response(&reply.text, &items) {Ok(x) => x,Err(e) => {tracing::warn!(%kb_id, error = %e, "类别词对齐回复解析失败,这一批留到下次");diff --git a/docs/design/extraction.md b/docs/design/extraction.mdindex 3357a8d5..0b8819e6 100644--- a/docs/design/extraction.md+++ b/docs/design/extraction.md@@ -79,6 +79,32 @@ reads 6.9% of statements as misworded against 3.7%, where judging the same graphnumber by 1.4. **The lever is measured and left alone.** The read timeout it was meant to answer isalready answered by streaming.+**Where a long answer stops** [#760]. The extraction and alignment calls send an explicit+`max_tokens` of 65,536. Without it the ceiling is whatever the endpoint defaults to: with reasoning+disabled a dense passage's answer stopped at exactly 4,096 completion tokens with+`finish_reason: length`, and that JSON could not be parsed.++The number is measured against the completion, not against the answer, because `max_tokens` does not+mean the same thing everywhere: Anthropic counts thinking inside it and OpenAI's+`max_completion_tokens` counts reasoning tokens. On an endpoint that counts reasoning, a ceiling+chosen to fit the answer would be a budget for the thinking instead, and that is the lever measured+and rejected under **What a reasoning cap costs** above: holding this model to about 16,000 reasoning+tokens cost 885 statements against 729, and took table figures reaching no statement from 2% to 16%.+The largest completion measured on this path is 45,428 tokens, so 65,536 sits above it and can only+ever guard against the endpoint's own default. The answer itself is about 1,500 tokens, nowhere near+either line.++Sending it is not a guarantee. The same endpoint ignores the field with reasoning on, where a call+capped at 4,096 returned 45,428 completion tokens and finished normally; it is honoured with+reasoning off, which is where the truncation was seen. The field costs nothing where it is not read.++`finish_reason` travels with the answer rather than being dropped once the stream is known to have+ended, because a reply cut at the ceiling is a valid string and looks complete to the parser. With+the reason carried through, a chunk that hits the ceiling says so — the drop row reads *hit the+token ceiling* instead of only *could not be parsed* — and the two causes stay apart: an answer that+is too long is a number to change, a reply that does not fit the shape is not. Across all bases in+the e2e database `truncated_reply` has fired 10 times, and those rows keep every complete item.+**Server checks, each a drop reason in `extraction_drops`.** Every quote must occur in the chunk(`quote_not_in_chunk`); every name in its quote (`name_not_in_text`); a time mention only when itswords occur in the statement's own quote (`time_not_in_quote`), because the model otherwise attaches