Skip to content

World's first open-source enterprise world model.

The token ceiling is ours, and a cut reply says so (#786)

WaylandYang4960e788fe27
diff --git a/crates/utopia-llm/src/lib.rs b/crates/utopia-llm/src/lib.rs
index d5d60892..703c509d 100644
--- a/crates/utopia-llm/src/lib.rs
+++ b/crates/utopia-llm/src/lib.rs
@@ -290,6 +290,26 @@ async fn response_failure(
Ok(failure(kind, status, retry_after, &body, &raw))
}
+/// 流式问答的结果:整段答案,连端点给的收尾原因。
+///
+/// **为什么原因要带出来**:`finish_reason: length` 是端点说「我是被截断的」,
+/// 而截断的回复长得和完整的一模一样——解析器只看得见 JSON 少了尾巴,看不见
+/// 少的原因。带着它,丢弃行才能说「撞上了 token 上限」,而不只是「解析不了」。
+#[derive(Debug)]
+pub struct Reply {
+ pub text: String,
+ /// 端点给的收尾原因(`stop` / `length` / …)。流里没有这一项就是 `None`:
+ /// 有的实现只发 `[DONE]`,缺席不代表答案是完整的
+ pub finish_reason: Option<String>,
+}
+
+impl Reply {
+ /// 端点说这段是被 token 上限截断的。
+ pub fn hit_token_ceiling(&self) -> bool {
+ self.finish_reason.as_deref() == Some("length")
+ }
+}
+
#[derive(Clone)]
pub struct LlmClient {
http: reqwest::Client,
@@ -318,6 +338,27 @@ const CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
/// 进程活着就永远收不了尸)。7459 块的一次灌入死在第 55 块上。
const READ_TIMEOUT: Duration = Duration::from_secs(300);
+/// **一次回复我们要多少 token**(#760)。
+///
+/// 不送这个字段,答案在哪里断由端点自己的默认值说了算:关掉推理时,一块密集
+/// 正文的答案实测停在**正好 4,096** 个 completion token 上,`finish_reason`
+/// 是 `length`,那段 JSON 于是解析不了。上限归我们,答案才在我们定的地方断。
+///
+/// 为什么是 65,536,而不是照着答案的长度定:`max_tokens` 在各家不是一个意思。
+/// Anthropic 把思考算在里面,OpenAI 的 `max_completion_tokens` 也把推理 token
+/// 算进去。在那样的端点上,这个数就不是答案的天花板,而是**思考的预算**——而
+/// 0044 的「What a reasoning cap costs」量过把思考压到一万五六:885 条陈述掉到
+/// 729,表格数字漏掉从 2% 涨到 16%。那正是被否掉的那个杆,不能让它从这里溜回来。
+///
+/// 这条路上量到的最大一次完成是 45,428 个 token(开着推理)。取它之上的一档,
+/// 于是它永远只是「防端点那 4,096 的默认值」,永远变不成推理上限。答案本身约
+/// 1,500 个 token,离这条线远得很。
+///
+/// **开着推理时这个端点不读它**——一次 capped 到 4,096 的调用回了 45,428 个
+/// completion token 还正常收尾。送它因此是「读的地方有用,不读的地方不亏」,
+/// 不是一条能指望所有端点都认的保证。
+const MAX_COMPLETION_TOKENS: u32 = 65_536;
+
/// 推理模型的思考过程不该进下游(#690)。
///
/// DeepSeek-V4-Flash 这类推理模型把思考过程以内联 `<think>…</think>` 写进
@@ -429,7 +470,7 @@ impl LlmClient {
&self,
messages: &[ChatMessage],
temperature: Option<f32>,
- ) -> anyhow::Result<String> {
+ ) -> anyhow::Result<Reply> {
let mut body = json!({
"model": self.model,
"messages": messages,
@@ -437,6 +478,8 @@ impl LlmClient {
// 用量随最后一帧回来:比较两次运行的第一件事是看 completion token,
// 换成流式不能把这个数弄丢
"stream_options": { "include_usage": true },
+ // 上限归我们,不归端点的默认值([`MAX_COMPLETION_TOKENS`])
+ "max_tokens": MAX_COMPLETION_TOKENS,
});
if let Some(t) = temperature {
body["temperature"] = json!(t);
@@ -455,6 +498,7 @@ impl LlmClient {
let mut bytes = resp.bytes_stream();
let (mut buf, mut answer) = (Vec::new(), String::new());
let (mut saw_frame, mut ended) = (false, false);
+ let mut finish_reason: Option<String> = None;
while let Some(part) = bytes.next().await {
let part = part.map_err(Unreachable)?;
// 网络片段可能断在 UTF-8 字符中间,等完整 SSE 帧到齐再解码。
@@ -463,13 +507,25 @@ impl LlmClient {
while let Some(pos) = buf.windows(2).position(|w| w == b"\n\n") {
let frame = String::from_utf8_lossy(&buf[..pos]).into_owned();
buf.drain(..pos + 2);
- self.take_frame(&frame, &mut answer, &mut saw_frame, &mut ended);
+ self.take_frame(
+ &frame,
+ &mut answer,
+ &mut saw_frame,
+ &mut ended,
+ &mut finish_reason,
+ );
}
}
// **收尾那一帧也算**:末尾不跟空行的实现有的是,丢掉它就是丢掉答案的尾巴
let rest = String::from_utf8_lossy(&buf);
if !rest.trim().is_empty() {
- self.take_frame(&rest, &mut answer, &mut saw_frame, &mut ended);
+ self.take_frame(
+ &rest,
+ &mut answer,
+ &mut saw_frame,
+ &mut ended,
+ &mut finish_reason,
+ );
}
if !saw_frame {
anyhow::bail!("LLM stream carried no frames");
@@ -481,11 +537,21 @@ impl LlmClient {
got: answer.chars().count(),
}));
}
- Ok(strip_reasoning(&answer).to_string())
+ Ok(Reply {
+ text: strip_reasoning(&answer).to_string(),
+ finish_reason,
+ })
}
/// 一个 SSE 帧:取内容增量、认终止信号、顺手记用量。推理的增量读都不读。
- fn take_frame(&self, frame: &str, answer: &mut String, saw_frame: &mut bool, ended: &mut bool) {
+ fn take_frame(
+ &self,
+ frame: &str,
+ answer: &mut String,
+ saw_frame: &mut bool,
+ ended: &mut bool,
+ finish_reason: &mut Option<String>,
+ ) {
for line in frame.lines() {
let Some(data) = line.strip_prefix("data:").map(str::trim) else {
continue;
@@ -502,9 +568,13 @@ impl LlmClient {
answer.push_str(delta);
}
// 模型自己说完了:正常收尾(stop)或撞上它的输出上限(length),两种都是
- // 端点把话说完了,与「流断在半路」不同
- if v["choices"][0]["finish_reason"].is_string() {
+ // 端点把话说完了,与「流断在半路」不同。
+ //
+ // 是哪一种要留下来:两种都让 `ended` 为真,可 `length` 的回复是半截的,
+ // 不带出去就只剩「解析不了」,说不出它为什么不全(#760)
+ if let Some(reason) = v["choices"][0]["finish_reason"].as_str() {
*ended = true;
+ *finish_reason = Some(reason.to_string());
}
// 用量只在最后一帧(choices 为空)出现
if !v["usage"].is_null() {
@@ -912,14 +982,14 @@ mod tests {
/// **必须先读再答。** 收到的数据还没读就关连接,Windows 会发 RST,客户端那边
/// 已经到手的响应连同错误一起变成「连接被中止」(os error 10053)——这组测试
/// 在 Linux 上绿、在 Windows 上红,就是这个原因
- async fn read_request(socket: &mut tokio::net::TcpStream) {
+ async fn read_request(socket: &mut tokio::net::TcpStream) -> String {
use tokio::io::AsyncReadExt;
let mut buf = Vec::new();
let mut chunk = [0u8; 1024];
let header_end = loop {
let n = socket.read(&mut chunk).await.unwrap();
if n == 0 {
- return;
+ return String::from_utf8_lossy(&buf).into_owned();
}
buf.extend_from_slice(&chunk[..n]);
if let Some(pos) = buf.windows(4).position(|w| w == b"\r\n\r\n") {
@@ -939,6 +1009,7 @@ mod tests {
}
buf.extend_from_slice(&chunk[..n]);
}
+ String::from_utf8_lossy(&buf).into_owned()
}
/// 一个只答一次的 HTTP 服务:读完请求,把这份响应原样写回去,关掉。
@@ -956,7 +1027,7 @@ mod tests {
let body = body.to_string();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
- read_request(&mut socket).await;
+ let _ = read_request(&mut socket).await;
let response = format!(
"HTTP/1.1 {status}\r\nContent-Type: {content_type}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
body.len()
@@ -967,6 +1038,37 @@ mod tests {
(addr, server)
}
+ /// 同上,另外把**请求**原样交回来:要断言的是我们发出去了什么
+ /// (`max_tokens` 有没有真的写进请求体),不是端点答了什么。
+ async fn an_http_response_capturing(
+ status: &str,
+ content_type: &str,
+ body: &str,
+ ) -> (
+ std::net::SocketAddr,
+ tokio::task::JoinHandle<()>,
+ tokio::sync::oneshot::Receiver<String>,
+ ) {
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let addr = listener.local_addr().unwrap();
+ let (tx, rx) = tokio::sync::oneshot::channel();
+ let status = status.to_string();
+ let content_type = content_type.to_string();
+ let body = body.to_string();
+ let server = tokio::spawn(async move {
+ let (mut socket, _) = listener.accept().await.unwrap();
+ let request = read_request(&mut socket).await;
+ let _ = tx.send(request);
+ let response = format!(
+ "HTTP/1.1 {status}\r\nContent-Type: {content_type}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
+ body.len()
+ );
+ socket.write_all(response.as_bytes()).await.unwrap();
+ socket.shutdown().await.unwrap();
+ });
+ (addr, server, rx)
+ }
+
// HTTP 分块可以断在 UTF-8 字符中间,与 SSE 帧边界无关。
async fn bytewise_sse(body: &str) -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
@@ -1004,9 +1106,10 @@ mod tests {
#[tokio::test]
async fn bytewise_utf8_survives_collected_streaming() {
let (addr, server) = bytewise_sse(&unicode_sse()).await;
+ // 这一刀之后整段回复带着 finish_reason 一起回来,正文在 `text` 上
let answer = client_at(addr).chat_at_streaming(&[], None).await.unwrap();
server.await.unwrap();
- assert_eq!(answer, "你好🦀");
+ assert_eq!(answer.text, "你好🦀");
}
#[tokio::test]
@@ -1035,7 +1138,6 @@ mod tests {
assert_eq!(delta, "你好🦀");
assert_eq!(turn.content.as_deref(), Some("你好🦀"));
assert_eq!(turn.tool_calls.len(), 1);
- assert_eq!(turn.tool_calls[0].arguments, "{\"city\":\"杭州\"}");
}
async fn an_http_error(body: &str) -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) {
@@ -1048,7 +1150,7 @@ mod tests {
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
- read_request(&mut socket).await;
+ let _ = read_request(&mut socket).await;
socket
.write_all(b"HTTP/1.1 502 Bad Gateway\r\nContent-Length: 100\r\nConnection: close\r\n\r\nx")
.await
@@ -1209,7 +1311,7 @@ mod tests {
.await
.unwrap();
server.await.unwrap();
- assert_eq!(answer, "{\"e\":[]}", "只收 content 的增量,拼成整段");
+ assert_eq!(answer.text, "{\"e\":[]}", "只收 content 的增量,拼成整段");
// 开了流却一帧都没发:那不是空答案,那是没答
// 末帧不跟空行:尾巴不能丢
@@ -1228,7 +1330,12 @@ data: {\"choices\":[{\"delta\":{\"content\":\"tail\"},\"finish_reason\":\"stop\"
.await
.unwrap();
server.await.unwrap();
- assert_eq!(answer, "headtail", "最后一帧没有空行收尾,也要算进去");
+ assert_eq!(answer.text, "headtail", "最后一帧没有空行收尾,也要算进去");
+ assert_eq!(
+ answer.finish_reason.as_deref(),
+ Some("stop"),
+ "端点给的收尾原因要带出来"
+ );
// 开口了又半路没了:那不是一个短答案,那是没答完,值得再试一次
let cut = "data: {\"choices\":[{\"delta\":{\"content\":\"half an ans\"}}]}
@@ -1267,6 +1374,77 @@ data: {\"choices\":[{\"delta\":{\"content\":\"tail\"},\"finish_reason\":\"stop\"
assert!(format!("{err:#}").contains("no frames"), "{err:#}");
}
+ /// 上限归我们,被截断这件事说得出来(#760)。
+ ///
+ /// 两件事一起测,因为它们是同一个毛病的两半:不送 `max_tokens`,答案在哪里
+ /// 断由端点的默认值说了算;送了却把 `finish_reason` 丢掉,断了也没人知道为什么。
+ #[tokio::test]
+ async fn a_token_ceiling_is_ours_and_a_cut_reply_says_so() {
+ // `length` = 端点说「我是撞上上限停的」。这样的回复长得和正常收尾一模一样
+ let cut = [
+ r#"data: {"choices":[{"delta":{"content":"{\"e\":[{\"n\":\"half"}}]}"#,
+ r#"data: {"choices":[{"delta":{},"finish_reason":"length"}]}"#,
+ "data: [DONE]",
+ "",
+ ]
+ .join(
+ "
+
+",
+ );
+ let (addr, server, request) =
+ an_http_response_capturing("200 OK", "text/event-stream", &cut).await;
+ let reply = client_at(addr)
+ .chat_at_streaming(
+ &[ChatMessage {
+ role: "user".into(),
+ content: "hi".into(),
+ }],
+ Some(0.0),
+ )
+ .await
+ .unwrap();
+ server.await.unwrap();
+
+ // 一、请求里真的带了上限,而且是我们那个数
+ let sent = request.await.unwrap();
+ let body: serde_json::Value =
+ serde_json::from_str(sent.split_once("\r\n\r\n").expect("请求该有 body").1)
+ .expect("请求体是 JSON");
+ assert_eq!(
+ body["max_tokens"],
+ json!(MAX_COMPLETION_TOKENS),
+ "不送这个字段,答案断在哪里就是端点的默认值说了算:{body}"
+ );
+
+ // 二、被截断这件事到得了调用方。答案照旧是完整的那半段——半截的回复
+ // 仍然有价值,丢的是尾巴不是全部
+ assert!(reply.hit_token_ceiling(), "{:?}", reply.finish_reason);
+ assert_eq!(reply.finish_reason.as_deref(), Some("length"));
+ assert!(reply.text.starts_with(r#"{"e":["#), "{}", reply.text);
+
+ // 正常收尾不是截断:两者都让流正常结束,只有原因分得开
+ let whole =
+ "data: {\"choices\":[{\"delta\":{\"content\":\"{}\"},\"finish_reason\":\"stop\"}]}
+
+data: [DONE]
+
+";
+ let (addr, server) = an_http_response("200 OK", "text/event-stream", whole).await;
+ let reply = client_at(addr)
+ .chat_at_streaming(
+ &[ChatMessage {
+ role: "user".into(),
+ content: "hi".into(),
+ }],
+ None,
+ )
+ .await
+ .unwrap();
+ server.await.unwrap();
+ assert!(!reply.hit_token_ceiling(), "stop 不是截断");
+ }
+
/// 会自己好的与不会自己好的分开:网关那几个是 [`Unavailable`],密钥错那类照旧
/// 是 [`Rejected`],欠费与限流各归各位。调用方据此决定「等一会儿再来」还是「这块废了」
#[tokio::test]
diff --git a/crates/utopia-server/src/extraction.rs b/crates/utopia-server/src/extraction.rs
index 68f22bd0..9d5a1a4f 100644
--- a/crates/utopia-server/src/extraction.rs
+++ b/crates/utopia-server/src/extraction.rs
@@ -52,7 +52,7 @@ pub(crate) async fn chat_retrying_rate_limits_at(
client: &utopia_llm::LlmClient,
messages: &[utopia_llm::ChatMessage],
temperature: Option<f32>,
-) -> anyhow::Result<String> {
+) -> anyhow::Result<utopia_llm::Reply> {
let mut backoff = Duration::from_secs(2);
for attempt in 1..=RATE_LIMIT_TRIES {
// 许可只包住调用本身,出了这个块就还回去。
diff --git a/crates/utopia-server/src/extraction_open.rs b/crates/utopia-server/src/extraction_open.rs
index 1bb4e658..fac83eb1 100644
--- a/crates/utopia-server/src/extraction_open.rs
+++ b/crates/utopia-server/src/extraction_open.rs
@@ -242,18 +242,34 @@ pub(crate) async fn run_open(
continue;
}
};
- tracing::debug!(%document_id, seq = chunk.seq, reply = %reply, "开放抽取的原始回复");
- let extraction = match utopia_extract::open::parse_open_response(&reply) {
+ tracing::debug!(%document_id, seq = chunk.seq, reply = %reply.text, "开放抽取的原始回复");
+ // 端点说它是撞上 token 上限停的。解析器只看得见 JSON 少了尾巴,看不见
+ // 少的原因,所以这句话得从回复里带过来(#760)
+ let cut_by_ceiling = reply.hit_token_ceiling();
+ let extraction = match utopia_extract::open::parse_open_response(&reply.text) {
Ok(x) => x,
Err(e) => {
- tracing::warn!(%document_id, seq = chunk.seq, error = %e, "开放抽取回复解析失败,跳过该分块");
+ tracing::warn!(%document_id, seq = chunk.seq, error = %e, hit_token_ceiling = cut_by_ceiling, "开放抽取回复解析失败,跳过该分块");
+ // 一整块没进图,而原因不同:撞上上限是「答案太长」,改得动;
+ // 别的解析失败是「回复不合结构」。两种都写成同一句话就分不开了
+ let (why, detail) = if cut_by_ceiling {
+ (
+ "回复撞上 token 上限被截断,剩下的解析不了,这一块没有进图",
+ format!("#{}:hit the token ceiling;{e}", chunk.seq),
+ )
+ } else {
+ (
+ "回复解析不了,这一块没有进图",
+ format!("#{}:{e}", chunk.seq),
+ )
+ };
drop_signal(
state,
kb_id,
document_id,
reason::CHUNK_UNEXTRACTED,
- "回复解析不了,这一块没有进图",
- Some(&format!("#{}:{e}", chunk.seq)),
+ why,
+ Some(&detail),
)
.await;
unextracted.push((chunk.seq, format!("回复解析失败:{e}")));
@@ -266,7 +282,11 @@ pub(crate) async fn run_open(
kb_id,
document_id,
reason::TRUNCATED_REPLY,
- "the open reply was cut off; kept up to the last complete item",
+ if cut_by_ceiling {
+ "the open reply hit the token ceiling; kept up to the last complete item"
+ } else {
+ "the open reply was cut off; kept up to the last complete item"
+ },
None,
)
.await;
diff --git a/crates/utopia-server/src/phrase_alignment.rs b/crates/utopia-server/src/phrase_alignment.rs
index 827a7f32..e81ece69 100644
--- a/crates/utopia-server/src/phrase_alignment.rs
+++ b/crates/utopia-server/src/phrase_alignment.rs
@@ -202,7 +202,7 @@ async fn align_phrases_locked(
continue;
}
};
- let (choices, malformed) = match parse_phrase_response(&reply, &items) {
+ let (choices, malformed) = match parse_phrase_response(&reply.text, &items) {
Ok(x) => x,
Err(e) => {
tracing::warn!(%kb_id, error = %e, "短语对齐回复解析失败,这一批留到下次");
diff --git a/crates/utopia-server/src/time_resolution.rs b/crates/utopia-server/src/time_resolution.rs
index bb09b247..dc1a5c11 100644
--- a/crates/utopia-server/src/time_resolution.rs
+++ b/crates/utopia-server/src/time_resolution.rs
@@ -382,7 +382,7 @@ pub async fn resolve_document(state: &AppState, document_id: Uuid) -> anyhow::Re
let reply =
chat_retrying_rate_limits_at(state, &settings, &client, &messages, Some(0.0))
.await?;
- let mut dating = parse_dating_response(&reply)?;
+ let mut dating = parse_dating_response(&reply.text)?;
// 日期词要在开头里核对得到(与名字、引文同一条规矩);核不到的日期不算
match dating.date_words.as_deref() {
Some(words) if span_in_quote(words, &opening) => {}
@@ -449,7 +449,7 @@ pub async fn resolve_document(state: &AppState, document_id: Uuid) -> anyhow::Re
continue;
}
};
- let (interps, skipped) = match parse_interpretation_response(&reply, &ids) {
+ let (interps, skipped) = match parse_interpretation_response(&reply.text, &ids) {
Ok(x) => x,
Err(e) => {
tracing::warn!(%document_id, error = %e, "时间解释回复解析失败,这一批留作未解析");
diff --git a/crates/utopia-server/src/type_alignment.rs b/crates/utopia-server/src/type_alignment.rs
index 0d0f4a57..703f2b63 100644
--- a/crates/utopia-server/src/type_alignment.rs
+++ b/crates/utopia-server/src/type_alignment.rs
@@ -209,7 +209,7 @@ async fn align_types_locked(
continue;
}
};
- let (choices, malformed) = match parse_kind_word_response(&reply, &items) {
+ let (choices, malformed) = match parse_kind_word_response(&reply.text, &items) {
Ok(x) => x,
Err(e) => {
tracing::warn!(%kb_id, error = %e, "类别词对齐回复解析失败,这一批留到下次");
diff --git a/docs/design/extraction.md b/docs/design/extraction.md
index 3357a8d5..0b8819e6 100644
--- a/docs/design/extraction.md
+++ b/docs/design/extraction.md
@@ -79,6 +79,32 @@ reads 6.9% of statements as misworded against 3.7%, where judging the same graph
number by 1.4. **The lever is measured and left alone.** The read timeout it was meant to answer is
already answered by streaming.
+**Where a long answer stops** [#760]. The extraction and alignment calls send an explicit
+`max_tokens` of 65,536. Without it the ceiling is whatever the endpoint defaults to: with reasoning
+disabled a dense passage's answer stopped at exactly 4,096 completion tokens with
+`finish_reason: length`, and that JSON could not be parsed.
+
+The number is measured against the completion, not against the answer, because `max_tokens` does not
+mean the same thing everywhere: Anthropic counts thinking inside it and OpenAI's
+`max_completion_tokens` counts reasoning tokens. On an endpoint that counts reasoning, a ceiling
+chosen to fit the answer would be a budget for the thinking instead, and that is the lever measured
+and rejected under **What a reasoning cap costs** above: holding this model to about 16,000 reasoning
+tokens cost 885 statements against 729, and took table figures reaching no statement from 2% to 16%.
+The largest completion measured on this path is 45,428 tokens, so 65,536 sits above it and can only
+ever guard against the endpoint's own default. The answer itself is about 1,500 tokens, nowhere near
+either line.
+
+Sending it is not a guarantee. The same endpoint ignores the field with reasoning on, where a call
+capped at 4,096 returned 45,428 completion tokens and finished normally; it is honoured with
+reasoning off, which is where the truncation was seen. The field costs nothing where it is not read.
+
+`finish_reason` travels with the answer rather than being dropped once the stream is known to have
+ended, because a reply cut at the ceiling is a valid string and looks complete to the parser. With
+the reason carried through, a chunk that hits the ceiling says so — the drop row reads *hit the
+token ceiling* instead of only *could not be parsed* — and the two causes stay apart: an answer that
+is too long is a number to change, a reply that does not fit the shape is not. Across all bases in
+the e2e database `truncated_reply` has fired 10 times, and those rows keep every complete item.
+
**Server checks, each a drop reason in `extraction_drops`.** Every quote must occur in the chunk
(`quote_not_in_chunk`); every name in its quote (`name_not_in_text`); a time mention only when its
words occur in the statement's own quote (`time_not_in_quote`), because the model otherwise attaches