Skip to content

World's first open-source enterprise world model.

Long conversations shed old exchanges and recover from context refusals (#973)

Maya Kide5b5681db34e
diff --git a/crates/utopia-llm/src/lib.rs b/crates/utopia-llm/src/lib.rs
index 6420f47e..62d11c9d 100644
--- a/crates/utopia-llm/src/lib.rs
+++ b/crates/utopia-llm/src/lib.rs
@@ -204,6 +204,36 @@ pub fn rejected(err: &anyhow::Error) -> Option<&Rejected> {
err.chain().find_map(|e| e.downcast_ref::<Rejected>())
}
+/// 提示超过了模型的上下文窗口——既不是「不支持工具」,也不是补全上限(#964)。
+///
+/// 文字保持中性并带着状态码:`failure` 是抽取、治理和对话共用的,一个超长的 chunk
+/// 不是「对话太长」;对话自己的那句话在界面文案里(错误码 `context_too_long`)
+#[derive(Debug, thiserror::Error)]
+#[error("LLM request failed ({reason}): the prompt exceeds the model's context window: {detail}")]
+pub struct ContextTooLong {
+ pub status: u16,
+ /// 状态码带原因短语,同 `Rejected`
+ pub reason: String,
+ pub detail: String,
+ /// 端点报出的窗口(token)。只在数字紧跟着「窗口」这个词、而且不小于
+ /// `MIN_CONTEXT_WINDOW_TOKENS` 时才有
+ pub window: Option<u32>,
+}
+
+/// 历史的缺省字符预算。DeepSeek 一次收下了 25 万 token 的提示(2026-09-27 实测),
+/// 这个数远在任何窗口之下;它管的是别把二十轮表格原样搬回去,不是贴着窗口走
+const DEFAULT_HISTORY_CHARS: usize = 32_000;
+/// 历史预算的下限:学到的窗口只降不升、一直用到设置改动或重启,一条报了小数字的
+/// 错误不能把历史饿死(#973 评审)
+const MIN_HISTORY_CHARS: usize = 2_000;
+/// 比这小的「窗口」不信,和 `stated_completion_ceiling` 一个道理:没有模型的窗口那么小,
+/// 那多半是别的数字
+const MIN_CONTEXT_WINDOW_TOKENS: u32 = 1024;
+
+pub fn context_too_long(err: &anyhow::Error) -> Option<&ContextTooLong> {
+ err.chain().find_map(|e| e.downcast_ref::<ContextTooLong>())
+}
+
/// `Retry-After` 的整数秒形态。
///
/// 规范还允许 HTTP-date,这里**不解析**:为一个很少有人发的头引一个日期库不划算,
@@ -265,6 +295,17 @@ fn failure(
detail,
});
}
+ if kind == "LLM"
+ && matches!(status.as_u16(), 400 | 413 | 422)
+ && says_context_too_long(body, &detail)
+ {
+ return anyhow::Error::new(ContextTooLong {
+ status: status.as_u16(),
+ reason: status.to_string(),
+ window: stated_context_window(&detail),
+ detail,
+ });
+ }
anyhow::Error::new(Rejected {
kind: kind.to_string(),
status: status.as_u16(),
@@ -333,6 +374,9 @@ pub struct LlmClient {
/// 上限后记在这里,同一个客户端(含它的克隆)此后都按它发,一个进程只吃一次拒绝
/// (#891,#892 评审)。只降不升
completion_ceiling: std::sync::Arc<std::sync::atomic::AtomicU32>,
+ /// 端点报过的上下文窗口(token),起初是 `u32::MAX`。同一个客户端(含它的克隆)此后
+ /// 都按它算历史预算,只降不升;对话客户端按工作区留着,所以下一轮也记得(#964)
+ context_window: std::sync::Arc<std::sync::atomic::AtomicU32>,
}
/// 建连多久算失败。
@@ -440,12 +484,32 @@ impl LlmClient {
api_key: api_key.map(String::from),
model: model.to_string(),
reasoning_effort: None,
+ context_window: std::sync::Arc::new(std::sync::atomic::AtomicU32::new(u32::MAX)),
completion_ceiling: std::sync::Arc::new(std::sync::atomic::AtomicU32::new(
MAX_COMPLETION_TOKENS,
)),
}
}
+ /// 历史的字符预算。不是分词器估算:端点报过窗口就取窗口的**一半**当字数——中文差不多
+ /// 一字一 token,按 token 数原样当字数会在小窗口上超出(#973 评审)——另一半留给提示、
+ /// 工具和回答;再夹在 `MIN_HISTORY_CHARS` 与 `DEFAULT_HISTORY_CHARS` 之间。估得不准的
+ /// 那一次由一轮一次的恢复兜着
+ pub fn history_char_budget(&self) -> usize {
+ let window = self
+ .context_window
+ .load(std::sync::atomic::Ordering::Relaxed);
+ ((window / 2) as usize).clamp(MIN_HISTORY_CHARS, DEFAULT_HISTORY_CHARS)
+ }
+
+ /// 端点在拒绝里说了自己的窗口就记住它,只降不升
+ pub fn remember_context_window(&self, err: &anyhow::Error) {
+ if let Some(window) = context_too_long(err).and_then(|e| e.window) {
+ self.context_window
+ .fetch_min(window, std::sync::atomic::Ordering::Relaxed);
+ }
+ }
+
fn request(&self, path: &str) -> reqwest::RequestBuilder {
let mut req = self.http.post(format!("{}{path}", self.base_url));
if let Some(key) = &self.api_key {
@@ -544,7 +608,10 @@ impl LlmClient {
}
let raw = resp.text().await.map_err(Unreachable)?;
let parsed = serde_json::from_str(&raw).unwrap_or_default();
- if status == reqwest::StatusCode::BAD_REQUEST && !lowered {
+ if status == reqwest::StatusCode::BAD_REQUEST
+ && !lowered
+ && !says_context_too_long(&parsed, &err_detail(&parsed, &raw))
+ {
if let Some(lower) = stated_completion_ceiling(&err_detail(&parsed, &raw), ceiling)
{
tracing::info!(model = %self.model, sent = ceiling, stated = lower, "端点说了自己的补全上限,按它重试并记住");
@@ -1059,6 +1126,70 @@ fn says_out_of_credit(body: &serde_json::Value) -> bool {
.any(|v| v == "insufficient_quota")
}
+/// 响应体说不说这是上下文超了。
+///
+/// **先看结构化的 code,再看措辞。** OpenAI 的 code 是 `context_length_exceeded`,措辞随
+/// 版本变;DeepSeek 的 code 只是笼统的 `invalid_request_error`,认的是那句
+/// 「This model's maximum context length is 1048576 tokens. However, you requested …」
+/// (2026-09-27 实测)。code 在 `err_detail` 只留 message 之前读,不然就丢了
+fn says_context_too_long(body: &serde_json::Value, detail: &str) -> bool {
+ [
+ body.pointer("/error/code"),
+ body.pointer("/error/type"),
+ body.get("code"),
+ ]
+ .into_iter()
+ .flatten()
+ .filter_map(|v| v.as_str())
+ .any(|code| code == "context_length_exceeded")
+ || context_wording(detail)
+}
+
+/// 各家说「超长」的话。只收实测过或文档写明的:OpenAI 与 DeepSeek 的 "maximum context
+/// length"、Anthropic 的 "prompt is too long"、LM Studio 的 "exceeds the available context"
+fn context_wording(detail: &str) -> bool {
+ let detail = detail.to_ascii_lowercase();
+ [
+ "maximum context length",
+ "context window",
+ "too many tokens",
+ "prompt is too long",
+ "exceeds the available context",
+ ]
+ .iter()
+ .any(|wording| detail.contains(wording))
+}
+
+/// 拒绝里报出的窗口有多大(token)。
+///
+/// **只认紧跟着「窗口」的那个数。** 同一句话里还有我们发了多少 token、模型的版本号、
+/// 补全上限,学错一个就把历史裁没了;数字后面还得跟着 tokens 这个词
+fn stated_context_window(detail: &str) -> Option<u32> {
+ let lower = detail.to_ascii_lowercase();
+ for marker in ["maximum context length", "context window"] {
+ let Some((_, rest)) = lower.split_once(marker) else {
+ continue;
+ };
+ let rest = rest.trim_start_matches(|c: char| c.is_whitespace() || matches!(c, ':' | '='));
+ let rest = rest
+ .strip_prefix("is ")
+ .or_else(|| rest.strip_prefix("of "))
+ .unwrap_or(rest);
+ let mut words = rest.split_whitespace();
+ let Some(number) = words.next().and_then(number_with_separators) else {
+ continue;
+ };
+ if number >= MIN_CONTEXT_WINDOW_TOKENS
+ && words
+ .next()
+ .is_some_and(|s| s.trim_end_matches(['.', ',', ';', ':']) == "tokens")
+ {
+ return Some(number);
+ }
+ }
+ None
+}
+
/// The ceiling the endpoint says it has, read out of the 400 it refused us with (#891).
///
/// `MAX_COMPLETION_TOKENS` is deliberately above the largest completion this
@@ -1081,11 +1212,12 @@ fn says_out_of_credit(body: &serde_json::Value) -> bool {
/// those at or above what we sent are the echo of our own request ("whereas you
/// provided 65536"); the largest of the rest is taken, and nothing below 1,024 is,
/// because no completion limit is that small and a stray small number would turn a
-/// refusal into a silently cut reply. A context-window refusal ("maximum context
-/// length is 32768 tokens") yields a number that may still be too large for the
-/// prompt; the retry then fails with the same 400, which is what happens today.
+/// refusal into a silently cut reply. A context-window refusal belongs to history
+/// recovery; treating its window as an output ceiling repeats the oversized prompt.
fn stated_completion_ceiling(detail: &str, sent: u32) -> Option<u32> {
- if !detail.contains("max_tokens") && !detail.contains("max_completion_tokens") {
+ if context_wording(detail)
+ || (!detail.contains("max_tokens") && !detail.contains("max_completion_tokens"))
+ {
return None;
}
let words: Vec<&str> = detail
@@ -1125,6 +1257,50 @@ fn number_with_separators(word: &str) -> Option<u32> {
#[cfg(test)]
mod tests {
+ #[test]
+ fn context_refusals_keep_the_code_and_only_learn_a_stated_window() {
+ // DeepSeek 的原话(2026-09-27 实测,deepseek-chat):code 只是 invalid_request_error
+ let deepseek = json!({"error":{"message":"This model's maximum context length is 1048576 tokens. However, you requested 1200006 tokens (1200005 in the messages, 1 in the completion). Please reduce the length of the messages or completion. (request_id: 3c27dcba)","type":"invalid_request_error","param":null,"code":"invalid_request_error"}});
+ for (body, raw, window, budget) in [
+ (json!({"error":{"code":"context_length_exceeded","message":"Request rejected"}}), "", None, 32_000),
+ (deepseek, "", Some(1_048_576), 32_000),
+ (json!({}), "Maximum context length is 16,384 tokens. You requested 24000 tokens for model v2026.", Some(16_384), 8_192),
+ (json!({}), "The context window is 8192 tokens", Some(8192), 4_096),
+ (json!({}), "The context window is 3000 tokens", Some(3000), 2_000),
+ (json!({}), "maximum context length is 512 tokens", None, 32_000),
+ (json!({}), "too many tokens: 123456", None, 32_000),
+ (json!({}), "Prompt is too long", None, 32_000),
+ (json!({}), "request (105140 tokens) exceeds the available context", None, 32_000),
+ ] {
+ let err = failure("LLM", reqwest::StatusCode::BAD_REQUEST, None, &body, raw).context("wrapped");
+ assert_eq!(context_too_long(&err).expect("context refusal").window, window);
+ assert!(rejected(&err).is_none());
+ assert!(transient(&err).is_none());
+ assert!(
+ err.root_cause().to_string().starts_with("LLM request failed (400 Bad Request): the prompt exceeds the model's context window: "),
+ "{err:#}"
+ );
+ let client = LlmClient::new("http://unused", None, "m");
+ client.remember_context_window(&err);
+ assert_eq!(client.clone().history_char_budget(), budget, "{raw}");
+ }
+ for raw in [
+ "tools are unsupported",
+ "max_tokens is too large: 16384 completion tokens",
+ "Invalid temperature",
+ ] {
+ let err = failure(
+ "LLM",
+ reqwest::StatusCode::BAD_REQUEST,
+ None,
+ &json!({}),
+ raw,
+ );
+ assert!(context_too_long(&err).is_none());
+ assert!(rejected(&err).is_some());
+ }
+ }
+
#[test]
fn reasoning_effort_rides_in_every_chat_body_only_when_set() {
let plain = LlmClient::new("http://x", None, "m");
@@ -1764,8 +1940,8 @@ data: {\"choices\":[{\"delta\":{\"content\":\"tail\"},\"finish_reason\":\"stop\"
);
assert_eq!(
stated_completion_ceiling("This model's maximum context length is 32768 tokens. However, you requested 65636 tokens (100 in the messages, 65536 in the completion). Please reduce the length of the messages or max_tokens.", sent),
- Some(32_768),
- "a context-window refusal lowers to the window; the retry may still fail, as today"
+ None,
+ "a context window is not a completion ceiling"
);
assert_eq!(
stated_completion_ceiling("max_completion_tokens must be at most 8 tokens", sent),
diff --git a/crates/utopia-server/src/api/chat.rs b/crates/utopia-server/src/api/chat.rs
index a05aa200..17a5d4b7 100644
--- a/crates/utopia-server/src/api/chat.rs
+++ b/crates/utopia-server/src/api/chat.rs
@@ -34,7 +34,6 @@ use uuid::Uuid;
use crate::auth::AuthUser;
use crate::error::ApiResult;
-use crate::llm_util;
use crate::retrieval;
use crate::state::AppState;
@@ -80,6 +79,8 @@ fn model_failure_code(err: &anyhow::Error) -> Option<&'static str> {
|| err.chain().any(|e| e.is::<utopia_llm::Interrupted>())
{
Some("model_unreachable")
+ } else if utopia_llm::context_too_long(err).is_some() {
+ Some("context_too_long")
} else if utopia_llm::rejected(err).is_some() {
Some("model_rejected")
} else {
@@ -657,7 +658,9 @@ pub async fn chat(
let settings = utopia_store::settings::get(&state.pool, kb.workspace_id)
.await?
.ok_or_else(|| AppError::invalid("no_chat_model", NO_MODEL))?;
- let client = llm_util::chat_client(&settings)
+ let client = state
+ .chat_clients
+ .get(kb.workspace_id, &settings)
.ok_or_else(|| AppError::invalid("no_chat_model", NO_MODEL))?;
let query = req.message.trim().to_string();
@@ -862,13 +865,6 @@ pub async fn chat(
let tool_server = ToolServer::new()
.dynamic_tools(agent::dynamic_tools(&shared))
.run();
- let rig_agent = AgentBuilder::new(RigModel::new(client.clone()))
- .preamble(&system_prompt)
- // 工具轮 + 最后那一轮作答;第 MAX_ROUNDS+1 次请求由钩子在 I/O 前交给纯作答阶段
- .default_max_turns(MAX_ROUNDS + 1)
- .add_hook(policy)
- .tool_server_handle(tool_server)
- .build();
let prior: Vec<(String, String)> = history
.turns
.iter()
@@ -876,6 +872,16 @@ pub async fn chat(
.filter(|(i, _)| Some(*i) != current)
.map(|(_, turn)| turn.clone())
.collect();
+ let context = super::chat_context::Context::new(
+ prior.clone(), history.last_tool_exchange.clone(), client.history_char_budget(),
+ );
+ let rig_agent = AgentBuilder::new(RigModel::with_context(client.clone(), context.clone()))
+ .preamble(&system_prompt)
+ // 工具轮 + 最后那一轮作答;第 MAX_ROUNDS+1 次请求由钩子在 I/O 前交给纯作答阶段
+ .default_max_turns(MAX_ROUNDS + 1)
+ .add_hook(policy)
+ .tool_server_handle(tool_server)
+ .build();
let mut runner = rig_agent
.runner(Message::user(query.clone()))
.history(agent::history_messages(&prior, &history.last_tool_exchange));
@@ -1063,7 +1069,7 @@ pub async fn chat(
kb_id,
workspace_id,
query.clone(),
- history.turns.clone(),
+ context.clone(),
client.clone(),
));
while let Some(frame) = legacy.next().await {
@@ -1083,12 +1089,13 @@ pub async fn chat(
let sink = shared.sink.lock().await;
(sink.sources.clone(), sink.resolved.clone())
};
+ let (bounded_history, bounded_exchange) = context.snapshot();
let input = finalization::AnswerContext {
- question: &query, history: &history.turns, current,
- prior_exchange: &history.last_tool_exchange,
+ question: &query, history: &bounded_history, current: None,
+ prior_exchange: &bounded_exchange,
exchange: &exchange_acc, sources: &sources, resolved: &resolved,
};
- match finalization::answer(&client, input).await {
+ match finalization::answer_with_context(&client, input, &context).await {
Ok(answer) => { turn_text = answer; turn_calls.clear(); finished = true; }
Err(e) => {
let message = format!("Model could not produce a final answer: {e}");
@@ -1204,7 +1211,7 @@ fn legacy_rag(
kb_id: Uuid,
workspace_id: Uuid,
query: String,
- turns: Vec<(String, String)>,
+ context: super::chat_context::Context,
client: utopia_llm::LlmClient,
) -> impl Stream<Item = ProducerEvent> {
async_stream::stream! {
@@ -1226,12 +1233,20 @@ fn legacy_rag(
"sources",
serde_json::to_string(&legacy_sources).unwrap_or_else(|_| "[]".into()),
));
- let mut lmsgs = vec![json!({ "role": "system", "content": legacy_system_prompt(&chunks) })];
- for (role, content) in &turns {
- lmsgs.push(json!({ "role": role, "content": content }));
- }
let mut answer_acc = String::new();
- match client.chat_stream_raw(&lmsgs).await {
+ let deltas = loop {
+ let (turns, _) = context.snapshot();
+ let mut lmsgs = vec![json!({ "role": "system", "content": legacy_system_prompt(&chunks) })];
+ for (role, content) in &turns {
+ lmsgs.push(json!({ "role": role, "content": content }));
+ }
+ lmsgs.push(json!({"role":"user", "content":query}));
+ match client.chat_stream_raw(&lmsgs).await {
+ Err(e) if context.recover(&client, &e) => continue,
+ result => break result,
+ }
+ };
+ match deltas {
Ok(deltas) => {
let mut deltas = std::pin::pin!(deltas);
while let Some(item) = deltas.next().await {
diff --git a/crates/utopia-server/src/api/chat_context.rs b/crates/utopia-server/src/api/chat_context.rs
new file mode 100644
index 00000000..8933815e
--- /dev/null
+++ b/crates/utopia-server/src/api/chat_context.rs
@@ -0,0 +1,294 @@
+//! 一整个用户轮次共用的一个历史窗口和一次上下文超限恢复(#964)。
+//!
+//! 能丢的只有此前已完成或已失败的整段问答,以及上一轮的工具往返;本轮的问题、系统
+//! 提示和本轮已经拿到的工具结果一个字都不动,工具也不重跑。落库的对话不变:裁的是
+//! 这一次发给模型的东西。
+use super::{agent, rig_model};
+use serde_json::Value;
+use std::sync::{Arc, Mutex};
+use utopia_llm::LlmClient;
+
+type Turns = Vec<(String, String)>;
+
+/// 一轮里三个地方——带工具的请求、RAG 兜底、只看证据的最终作答——共用同一个窗口,
+/// 所以是 `Arc<Mutex<_>>`:谁先撞上超限谁来裁,裁完别处照着发,一轮合计只裁一次
+#[derive(Clone)]
+pub(super) struct Context(Arc<Mutex<Window>>);
+
+struct Window {
+ /// 落库的历史,最旧在前;`start` 之前的不再发
+ turns: Turns,
+ /// 上一轮的工具往返:带 tool_calls 的助手消息和配套的 tool 结果,协议 JSON
+ exchange: Vec<Value>,
+ /// 最后一条助手消息在 `turns` 里的下标,往返插在它前面
+ last_assistant: Option<usize>,
+ start: usize,
+ /// 往返还发不发。**它单独一位,而且先于任何问答被丢**:一条 get_document 的结果就能
+ /// 有 24,000 字,两条就超过缺省预算。从前它和最后那条回答绑在一起,丢它就得把最旧
+ /// 到最新的问答一起丢,于是「再短一点」这样的追问不带任何历史发出去(#973 评审)。
+ /// 回答是人读过的对话,往返只是那条回答消化过的证据
+ exchange_kept: bool,
+ /// 这一轮恢复过没有,三个地方合计
+ retried: bool,
+}
+
+impl Window {
+ fn size(&self) -> usize {
+ self.turns[self.start..]
+ .iter()
+ .map(|(_, s)| s.chars().count())
+ .sum::<usize>()
+ + if self.keeps_exchange() {
+ self.exchange
+ .iter()
+ .map(|m| m.to_string().chars().count())
+ .sum()
+ } else {
+ 0
+ }
+ }
+
+ /// 往返随它的回答走:回答被裁掉了,往返也不能单独留下
+ fn keeps_exchange(&self) -> bool {
+ self.exchange_kept && self.last_assistant.is_some_and(|i| i >= self.start)
+ }
+
+ /// 把 `start` 推到下一条用户消息。按条数截的历史可能从一段问答的中间开始,
+ /// 落单的助手回答不发,哪怕预算装得下
+ fn skip_orphans(&mut self) {
+ while self.start < self.turns.len() && self.turns[self.start].0 != "user" {
+ self.start += 1;
+ }
+ }
+
+ /// 先丢往返,再从最旧的问答起整段整段地丢,直到装进预算;一条消息从不截半
+ fn trim(&mut self, budget: usize) {
+ self.skip_orphans();
+ if self.size() > budget && self.keeps_exchange() {
+ self.exchange_kept = false;
+ }
+ while self.start < self.turns.len() && self.size() > budget {
+ self.start += 1;
+ self.skip_orphans();
+ }
+ }
+}
+
+/// 落库的历史和往返展开成协议消息,和 rig 的 `wire` 走同一条路,数出来的条数才对得上
+fn protocol(turns: &[(String, String)], exchange: &[Value]) -> Vec<Value> {
+ let mut out = Vec::new();
+ for m in agent::history_messages(turns, exchange) {
+ rig_model::push_message(&mut out, &m);
+ }
+ out
+}
+
+impl Context {
+ pub fn new(turns: Turns, exchange: Vec<Value>, budget: usize) -> Self {
+ let last_assistant = turns.iter().rposition(|(role, _)| role == "assistant");
+ let mut window = Window {
+ turns,
+ exchange,
+ last_assistant,
+ start: 0,
+ exchange_kept: true,
+ retried: false,
+ };
+ window.trim(budget);
+ Self(Arc::new(Mutex::new(window)))
+ }
+
+ /// 现在还发的历史和往返
+ pub fn snapshot(&self) -> (Turns, Vec<Value>) {
+ let w = self.0.lock().unwrap();
+ (
+ w.turns[w.start..].to_vec(),
+ if w.keeps_exchange() {
+ w.exchange.clone()
+ } else {
+ vec![]
+ },
+ )
+ }
+
+ /// rig 的消息列表整轮不变:开头是它当初拿到的完整历史,后面跟本轮的问题和本轮的工具
+ /// 往返。每次发送前把历史那一段换成裁过的。数的是协议消息而不是对话轮次——一条落库
+ /// 的回答能展开成好几条工具消息;system 消息原地不动——认下的实体那条 system 就贴在
+ /// 问题前面。上一轮的往返在最后那条回答之前,丢它是从中间抠掉,不是掐头
+ pub fn apply(&self, messages: &[Value]) -> Vec<Value> {
+ let w = self.0.lock().unwrap();
+ let full = protocol(&w.turns, &w.exchange);
+ let kept = protocol(
+ &w.turns[w.start..],
+ if w.keeps_exchange() { &w.exchange } else { &[] },
+ );
+ let mut out = Vec::with_capacity(messages.len());
+ let mut replaced = 0;
+ for m in messages {
+ if m["role"] == "system" || replaced == full.len() {
+ out.push(m.clone());
+ continue;
+ }
+ replaced += 1;
+ if replaced == full.len() {
+ out.extend(kept.iter().cloned());
+ }
+ }
+ out
+ }
+
+ /// 端点说上下文超了:记住它报的窗口,把历史裁到一半再发一次,同一轮只来一次。
+ /// **裁不动就不重发**——历史已经空了、问题本身就超长的时候,原样再发只会再挨一次
+ /// 同样的 400(#973 评审)
+ pub fn recover(&self, client: &LlmClient, err: &anyhow::Error) -> bool {
+ if utopia_llm::context_too_long(err).is_none() {
+ return false;
+ }
+ client.remember_context_window(err);
+ let mut w = self.0.lock().unwrap();
+ if w.retried {
+ return false;
+ }
+ w.retried = true;
+ let before = (w.start, w.keeps_exchange());
+ let budget = (w.size() / 2).min(client.history_char_budget());
+ w.trim(budget);
+ if (w.start, w.keeps_exchange()) == before {
+ tracing::info!("Context refused and there is no older exchange to drop");
+ return false;
+ }
+ tracing::info!(
+ history_chars = w.size(),
+ "Context refused; retrying once with older exchanges removed"
+ );
+ true
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ fn refusal() -> anyhow::Error {
+ anyhow::Error::new(utopia_llm::ContextTooLong {
+ status: 400,
+ reason: "400 Bad Request".into(),
+ detail: "Prompt is too long".into(),
+ window: None,
+ })
+ }
+
+ fn heavy_exchange() -> Vec<Value> {
+ vec![
+ json!({"role":"assistant","tool_calls":[
+ {"id":"old-1","type":"function","function":{"name":"get_document","arguments":"{}"}},
+ {"id":"old-2","type":"function","function":{"name":"get_document","arguments":"{}"}}]}),
+ json!({"role":"tool","tool_call_id":"old-1","content":"x".repeat(24_000)}),
+ json!({"role":"tool","tool_call_id":"old-2","content":"y".repeat(24_000)}),
+ ]
+ }
+
+ #[test]
+ fn a_budget_drops_whole_exchanges_including_failed_and_stopped_turns() {
+ let turns = vec![
+ ("assistant".into(), "orphan".into()),
+ ("user".into(), "failed unanswered question".into()),
+ ("user".into(), "latest".into()),
+ ("assistant".into(), "[stopped]界😀".into()),
+ ];
+ let budget = "latest[stopped]界😀".chars().count();
+ let context = Context::new(turns.clone(), vec![], budget);
+ assert_eq!(context.snapshot().0, turns[2..]);
+ let smaller = Context::new(turns, vec![], budget - 1);
+ assert!(
+ smaller.snapshot().0.is_empty(),
+ "never cut the answer to make it fit"
+ );
+ }
+
+ #[test]
+ fn the_previous_tool_exchange_goes_before_any_question_or_answer() {
+ let turns: Turns = vec![
+ ("user".into(), "which documents mention Aurora?".into()),
+ (
+ "assistant".into(),
+ "Two documents mention Aurora [1][2].".into(),
+ ),
+ ];
+ let exchange = heavy_exchange();
+ let context = Context::new(turns.clone(), exchange.clone(), 32_000);
+ assert_eq!(context.snapshot().0, turns, "the question and answer stay");
+ assert!(context.snapshot().1.is_empty(), "the exchange goes first");
+
+ let system = json!({"role":"system","content":"preserve system"});
+ let entities = json!({"role":"system","content":"known entities"});
+ let current = json!({"role":"user","content":"make it shorter"});
+ let current_call = json!({"role":"assistant","tool_calls":[{"id":"new-call"}]});
+ let current_result =
+ json!({"role":"tool","tool_call_id":"new-call","content":"preserve evidence"});
+ let mut messages = vec![system.clone()];
+ messages.extend(protocol(&turns, &exchange));
+ messages.extend([
+ entities.clone(),
+ current.clone(),
+ current_call.clone(),
+ current_result.clone(),
+ ]);
+ let mut expected = vec![system];
+ expected.extend(protocol(&turns, &[]));
+ expected.extend([entities, current, current_call, current_result]);
+ assert_eq!(context.apply(&messages), expected);
+
+ let smaller = Context::new(turns.clone(), exchange, 10);
+ assert!(smaller.snapshot().0.is_empty(), "then the oldest exchanges");
+ assert!(smaller.snapshot().1.is_empty());
+ }
+
+ #[test]
+ fn a_small_exchange_stays_with_its_answer() {
+ let turns: Turns = vec![
+ ("user".into(), "old".into()),
+ ("assistant".into(), "answer".into()),
+ ("user".into(), "failed".into()),
+ ];
+ let exchange = vec![
+ json!({"role":"assistant","tool_calls":[{"id":"c","type":"function","function":{"name":"find_entities","arguments":"{}"}}]}),
+ json!({"role":"tool","tool_call_id":"c","content":"small"}),
+ ];
+ let context = Context::new(turns.clone(), exchange.clone(), 32_000);
+ assert_eq!(context.snapshot(), (turns.clone(), exchange.clone()));
+ let messages = protocol(&turns, &exchange);
+ assert_eq!(context.apply(&messages), messages, "nothing to remove");
+ }
+
+ #[test]
+ fn a_refusal_halves_the_history_once_and_only_when_something_can_go() {
+ let client = LlmClient::new("http://unused", None, "m");
+ let turns: Turns = (0..4)
+ .flat_map(|i| {
+ [
+ ("user".into(), format!("question {i}")),
+ ("assistant".into(), "界".repeat(1000)),
+ ]
+ })
+ .collect();
+ let context = Context::new(turns.clone(), heavy_exchange(), 32_000);
+ assert!(context.snapshot().1.is_empty());
+ assert!(context.recover(&client, &refusal()));
+ let (kept, _) = context.snapshot();
+ assert_eq!(kept, turns[4..], "half of four exchanges, the newest kept");
+ assert!(
+ !context.recover(&client, &refusal()),
+ "the second refusal of a turn is final"
+ );
+
+ let empty = Context::new(vec![], vec![], 32_000);
+ assert!(
+ !empty.recover(&client, &refusal()),
+ "nothing to drop, so the same request is not sent again"
+ );
+ let other = anyhow::anyhow!("network down");
+ assert!(!Context::new(turns, vec![], 32_000).recover(&client, &other));
+ }
+}
diff --git a/crates/utopia-server/src/api/chat_context_tests.rs b/crates/utopia-server/src/api/chat_context_tests.rs
new file mode 100644
index 00000000..789f20df
--- /dev/null
+++ b/crates/utopia-server/src/api/chat_context_tests.rs
@@ -0,0 +1,344 @@
+//! 长对话(#964):真处理函数、真库,端点是照脚本回话的假模型,不花钱也不分词。
+//!
+//! 每条测试看的是发给模型的请求:历史裁到多少、问题在不在、往返丢没丢、重发了几次。
+use super::*;
+
+const REFUSAL: &str = r#"{"error":{"code":"context_length_exceeded","message":"maximum context length is 4096 tokens; request has 12000 tokens"}}"#;
+const GATE: Reply = Reply::Tool(
+ "no_evidence_needed",
+ r#"{"reason":"restating the conversation"}"#,
+);
+
+async fn seed(f: &Fx, count: usize, chars: usize) -> anyhow::Result<Uuid> {
+ let id = utopia_store::conversations::create(&f.pool, f.kb, f.user.id, "context test").await?;
+ for i in 0..count {
+ for (role, text) in [
+ ("user", format!("question {i}")),
+ ("assistant", format!("answer {i}: {}", "界".repeat(chars))),
+ ] {
+ utopia_store::conversations::append_message(
+ &f.pool,
+ id,
+ role,
+ &text,
+ &utopia_store::conversations::TurnRecord::empty(),
+ )
+ .await?;
+ }
+ }
+ Ok(id)
+}
+
+fn history_chars(request: &serde_json::Value, current: &str) -> usize {
+ request["messages"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .filter(|m| m["role"] != "system" && m["content"] != current)
+ .filter_map(|m| m["content"].as_str())
+ .map(|s| s.chars().count())
+ .sum()
+}
+
+fn preserves_question(request: &serde_json::Value, current: &str) {
+ let messages = request["messages"].as_array().unwrap();
+ assert_eq!(
+ messages
+ .iter()
+ .filter(|m| m["role"] == "user" && m["content"] == current)
+ .count(),
+ 1
+ );
+ assert_eq!(messages[0]["role"], "system");
+ let first = messages.iter().find(|m| m["role"] != "system").unwrap();
+ assert_eq!(
+ first["role"], "user",
+ "no orphan answer at the history boundary"
+ );
+}
+
+#[tokio::test]
+async fn long_history_is_bounded_before_the_first_request() -> anyhow::Result<()> {
+ let Some(f) = fixture(Scripted::new(vec![GATE, Reply::Text("Recovered.")])).await? else {
+ return Ok(());
+ };
+ let id = seed(&f, 10, 6000).await?;
+ let sse = history_tests::ask_in(&f, id, "current question").await?;
+ assert!(sse.contains("event: done"), "{sse}");
+ let requests = f.requests();
+ assert_eq!(requests.len(), 2);
+ assert!(history_chars(&requests[0], "current question") <= 32_000);
+ assert!(!requests[0].to_string().contains("question 0"));
+ assert!(requests[0].to_string().contains("question 9"));
+ preserves_question(&requests[0], "current question");
+ // 裁的是这一次发出去的,落库的对话一条不少
+ let count: i64 =
+ sqlx::query_scalar("SELECT count(*) FROM conversation_messages WHERE conversation_id=$1")
+ .bind(id)
+ .fetch_one(&f.pool)
+ .await?;
+ assert_eq!(count, 22);
+ f.cleanup().await
+}
+
+#[tokio::test]
+async fn overflow_trims_once_and_the_next_turn_remembers_the_window() -> anyhow::Result<()> {
+ let Some(f) = fixture(Scripted::new(vec![
+ Reply::HttpError(400, REFUSAL),
+ GATE,
+ Reply::Text("Recovered."),
+ GATE,
+ Reply::Text("Next answer."),
+ ]))
+ .await?
+ else {
+ return Ok(());
+ };
+ let id = seed(&f, 6, 1500).await?;
+ let sse = history_tests::ask_in(&f, id, "current question").await?;
+ assert!(sse.contains("event: done"), "{sse}");
+ let requests = f.requests();
+ assert_eq!(requests.len(), 3);
+ assert!(
+ history_chars(&requests[1], "current question")
+ <= history_chars(&requests[0], "current question") / 2
+ );
+ assert_eq!(requests[0]["tools"], requests[1]["tools"]);
+ assert_eq!(requests[0]["tool_choice"], requests[1]["tool_choice"]);
+ assert_eq!(requests[0]["messages"][0], requests[1]["messages"][0]);
+ for r in &requests {
+ preserves_question(r, "current question");
+ }
+ let sse = history_tests::ask_in(&f, id, "next question").await?;
+ assert!(sse.contains("event: done"), "{sse}");
+ let requests = f.requests();
+ assert_eq!(requests.len(), 5);
+ // 端点说窗口是 4096 token:预算是它的一半,2048 字。最新那段问答加上「question 5」
+ // 那段装得下,再加「question 4」那段就超了
+ assert!(history_chars(&requests[3], "next question") <= 2048);
+ let sent = requests[3].to_string();
+ assert!(
+ sent.contains("question 5") && !sent.contains("question 4"),
+ "{sent}"
+ );
+ preserves_question(&requests[3], "next question");
+ // 换了模型、换了工作区,都不能继承这次拒绝
+ let ws: Uuid = sqlx::query_scalar("SELECT workspace_id FROM knowledge_bases WHERE id=$1")
+ .bind(f.kb)
+ .fetch_one(&f.pool)
+ .await?;
+ let mut settings = utopia_store::settings::get(&f.pool, ws).await?.unwrap();
+ assert_eq!(
+ f.state
+ .chat_clients
+ .get(ws, &settings)
+ .unwrap()
+ .history_char_budget(),
+ 2048
+ );
+ assert_eq!(
+ f.state
+ .chat_clients
+ .get(Uuid::now_v7(), &settings)
+ .unwrap()
+ .history_char_budget(),
+ 32_000
+ );
+ settings.chat_model = Some("changed-model".into());
+ assert_eq!(
+ f.state
+ .chat_clients
+ .get(ws, &settings)
+ .unwrap()
+ .history_char_budget(),
+ 32_000
+ );
+ f.cleanup().await
+}
+
+#[tokio::test]
+async fn a_second_refusal_has_its_own_code_and_never_falls_back() -> anyhow::Result<()> {
+ let Some(f) = fixture(Scripted::new(vec![
+ Reply::HttpError(400, REFUSAL),
+ Reply::HttpError(422, REFUSAL),
+ ]))
+ .await?
+ else {
+ return Ok(());
+ };
+ let id = seed(&f, 4, 2000).await?;
+ let sse = history_tests::ask_in(&f, id, "current question").await?;
+ assert!(sse.contains("event: error"), "{sse}");
+ assert!(sse.contains("context_too_long"), "{sse}");
+ assert!(!sse.contains("event: done"));
+ let requests = f.requests();
+ assert_eq!(requests.len(), 2);
+ for r in requests {
+ assert!(r.get("tools").is_some());
+ preserves_question(&r, "current question");
+ }
+ let count: i64 = sqlx::query_scalar(
+ "SELECT count(*) FROM conversation_messages WHERE conversation_id=$1 AND role='assistant'",
+ )
+ .bind(id)
+ .fetch_one(&f.pool)
+ .await?;
+ assert_eq!(count, 4, "no failed answer is stored as success");
+ f.cleanup().await
+}
+
+#[tokio::test]
+async fn overflow_after_a_tool_keeps_its_result_without_executing_it_again() -> anyhow::Result<()> {
+ let Some(f) = fixture(Scripted::new(vec![
+ GATE,
+ Reply::HttpError(400, REFUSAL),
+ Reply::Text("Recovered after the tool."),
+ ]))
+ .await?
+ else {
+ return Ok(());
+ };
+ let id = seed(&f, 4, 2000).await?;
+ let sse = history_tests::ask_in(&f, id, "current question").await?;
+ assert!(sse.contains("event: done"), "{sse}");
+ let requests = f.requests();
+ assert_eq!(requests.len(), 3);
+ let tools = |r: &serde_json::Value| {
+ r["messages"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .filter(|m| m["role"] == "tool" || m.get("tool_calls").is_some())
+ .cloned()
+ .collect::<Vec<_>>()
+ };
+ assert_eq!(tools(&requests[1]), tools(&requests[2]));
+ assert_eq!(tools(&requests[2]).len(), 2);
+ preserves_question(&requests[2], "current question");
+ f.cleanup().await
+}
+
+#[tokio::test]
+async fn rag_uses_the_same_bound_and_can_recover_without_changing_its_system() -> anyhow::Result<()>
+{
+ let Some(f) = fixture(Scripted::new(vec![
+ Reply::Http(400),
+ Reply::Http(400),
+ Reply::HttpError(400, REFUSAL),
+ Reply::Text("Fallback recovered."),
+ ]))
+ .await?
+ else {
+ return Ok(());
+ };
+ let id = seed(&f, 10, 6000).await?;
+ let sse = history_tests::ask_in(&f, id, "current question").await?;
+ assert!(sse.contains("event: done"), "{sse}");
+ let requests = f.requests();
+ assert_eq!(requests.len(), 4);
+ assert!(requests[2].get("tools").is_none());
+ assert!(history_chars(&requests[2], "current question") <= 32_000);
+ assert!(history_chars(&requests[3], "current question") <= 2048);
+ assert_eq!(requests[2]["messages"][0], requests[3]["messages"][0]);
+ for r in &requests {
+ preserves_question(r, "current question");
+ }
+ f.cleanup().await
+}
+
+#[tokio::test]
+async fn even_an_oversized_current_question_is_never_cut() -> anyhow::Result<()> {
+ let Some(f) = fixture(Scripted::new(vec![Reply::HttpError(400, REFUSAL)])).await? else {
+ return Ok(());
+ };
+ let question = "新问题😀".repeat(10_000);
+ let sse = f.ask(&question).await?;
+ assert!(sse.contains("context_too_long"), "{sse}");
+ let requests = f.requests();
+ // 没有历史可丢,同一个请求不再发第二遍
+ assert_eq!(requests.len(), 1);
+ preserves_question(&requests[0], &question);
+ f.cleanup().await
+}
+
+/// 上一轮的往返:读了两份文档,各 `chars` 字
+fn two_documents(chars: usize) -> serde_json::Value {
+ serde_json::json!([
+ {"role":"assistant","content":null,"tool_calls":[
+ {"id":"old-1","type":"function","function":{"name":"get_document","arguments":"{\"document_id\":\"a\"}"}},
+ {"id":"old-2","type":"function","function":{"name":"get_document","arguments":"{\"document_id\":\"b\"}"}}]},
+ {"role":"tool","tool_call_id":"old-1","content":"界".repeat(chars)},
+ {"role":"tool","tool_call_id":"old-2","content":"界".repeat(chars)}
+ ])
+}
+
+/// 一段落库的问答,回答带着 `two_documents` 的往返
+async fn seed_turn(f: &Fx, chars: usize) -> anyhow::Result<Uuid> {
+ let id = utopia_store::conversations::create(&f.pool, f.kb, f.user.id, "context test").await?;
+ utopia_store::conversations::append_message(
+ &f.pool,
+ id,
+ "user",
+ "which documents mention Aurora?",
+ &utopia_store::conversations::TurnRecord::empty(),
+ )
+ .await?;
+ utopia_store::conversations::append_message(
+ &f.pool,
+ id,
+ "assistant",
+ "Two documents mention Aurora [1][2].",
+ &utopia_store::conversations::TurnRecord {
+ tool_exchange: two_documents(chars),
+ ..utopia_store::conversations::TurnRecord::empty()
+ },
+ )
+ .await?;
+ Ok(id)
+}
+
+/// 上一轮读了两份文档,往返 48,000 字。追问「短一点」时往返被丢,上一轮的问答都还在;
+/// 往返装得下的时候照旧发
+#[tokio::test]
+async fn an_evidence_heavy_turn_still_leaves_its_question_and_answer_for_the_follow_up(
+) -> anyhow::Result<()> {
+ let Some(f) = fixture(Scripted::new(vec![GATE, Reply::Text("Shorter.")])).await? else {
+ return Ok(());
+ };
+ let id = seed_turn(&f, 24_000).await?;
+ let sse = history_tests::ask_in(&f, id, "make it shorter").await?;
+ assert!(sse.contains("event: done"), "{sse}");
+ let requests = f.requests();
+ let messages = requests[0]["messages"].as_array().unwrap();
+ assert!(messages
+ .iter()
+ .any(|m| m["role"] == "user" && m["content"] == "which documents mention Aurora?"));
+ assert!(messages.iter().any(
+ |m| m["role"] == "assistant" && m["content"] == "Two documents mention Aurora [1][2]."
+ ));
+ assert!(
+ !messages
+ .iter()
+ .any(|m| m["role"] == "tool" || m.get("tool_calls").is_some()),
+ "the exchange is dropped before the question and answer:\n{}",
+ requests[0]
+ );
+ preserves_question(&requests[0], "make it shorter");
+ f.cleanup().await?;
+
+ let Some(g) = fixture(Scripted::new(vec![GATE, Reply::Text("Shorter.")])).await? else {
+ return Ok(());
+ };
+ let id = seed_turn(&g, 100).await?;
+ let sse = history_tests::ask_in(&g, id, "make it shorter").await?;
+ assert!(sse.contains("event: done"), "{sse}");
+ let requests = g.requests();
+ let messages = requests[0]["messages"].as_array().unwrap();
+ assert_eq!(
+ messages.iter().filter(|m| m["role"] == "tool").count(),
+ 2,
+ "a small exchange is still sent:\n{}",
+ requests[0]
+ );
+ g.cleanup().await
+}
diff --git a/crates/utopia-server/src/api/chat_empty_reply_tests.rs b/crates/utopia-server/src/api/chat_empty_reply_tests.rs
index 88ec03e2..6cc889f5 100644
--- a/crates/utopia-server/src/api/chat_empty_reply_tests.rs
+++ b/crates/utopia-server/src/api/chat_empty_reply_tests.rs
@@ -39,6 +39,7 @@ pub(super) enum Reply {
OversizedText,
Finished(&'static str, &'static str),
Http(u16),
+ HttpError(u16, &'static str),
/// 调一个工具:(名字, 参数 JSON)
Tool(&'static str, &'static str),
}
@@ -72,6 +73,9 @@ impl Respond for Scripted {
seen.len()
};
let frame = match self.replies.get(n - 1).copied().unwrap_or(Reply::Empty) {
+ Reply::HttpError(status, body) => {
+ return ResponseTemplate::new(status).set_body_string(body)
+ }
Reply::Http(status) => {
return ResponseTemplate::new(status).set_body_string("Evidence gathering complete")
}
@@ -371,6 +375,8 @@ async fn a_reply_that_stays_empty_is_an_error_after_one_retry() -> anyhow::Resul
f.cleanup().await
}
+#[path = "chat_context_tests.rs"]
+mod context_tests;
#[path = "chat_embed_once_tests.rs"]
mod embed_once_tests;
#[path = "chat_error_code_tests.rs"]
diff --git a/crates/utopia-server/src/api/chat_finalization.rs b/crates/utopia-server/src/api/chat_finalization.rs
index 4088810f..d56e080f 100644
--- a/crates/utopia-server/src/api/chat_finalization.rs
+++ b/crates/utopia-server/src/api/chat_finalization.rs
@@ -139,21 +139,38 @@ async fn answer_once(
anyhow::bail!("LLM stream ended unexpectedly")
}
-pub(super) async fn answer(client: &LlmClient, input: AnswerContext<'_>) -> anyhow::Result<String> {
- answer_with_deadline(client, input, ANSWER_DEADLINE).await
+pub(super) async fn answer_with_context(
+ client: &LlmClient,
+ input: AnswerContext<'_>,
+ context: &super::super::chat_context::Context,
+) -> anyhow::Result<String> {
+ answer_with_deadline(client, input, ANSWER_DEADLINE, Some(context)).await
}
async fn answer_with_deadline(
client: &LlmClient,
input: AnswerContext<'_>,
deadline: Duration,
+ context: Option<&super::super::chat_context::Context>,
) -> anyhow::Result<String> {
let mut messages = messages(&input);
// A total deadline covers BOTH attempts, not a fresh allowance per retry.
tokio::time::timeout(deadline, async {
for attempt in 1..=2 {
tracing::info!(attempt, "Requesting evidence-only final answer");
- match answer_once(client, &messages, input.question).await? {
+ let candidate = loop {
+ match answer_once(client, &messages, input.question).await {
+ Err(e) if context.is_some_and(|c| c.recover(client, &e)) => {
+ let (history, prior_exchange) = context.unwrap().snapshot();
+ let bounded = AnswerContext { history: &history, current: None, prior_exchange: &prior_exchange, ..input };
+ let refreshed = self::messages(&bounded);
+ // 只换背景那一条:第二次尝试时修正指令还在
+ messages[1] = refreshed[1].clone();
+ }
+ result => break result?,
+ }
+ };
+ match candidate {
Candidate::Accepted(text) => return Ok(text),
Candidate::Repairable(reason) if attempt == 1 => {
// Only the reason category crosses the boundary, never rejected prose.
@@ -170,6 +187,70 @@ async fn answer_with_deadline(
#[cfg(test)]
mod tests {
use super::*;
+ #[tokio::test]
+ async fn context_recovery_changes_only_background_and_shares_the_turn_retry() {
+ use std::sync::{Arc, Mutex};
+ use wiremock::{Mock, MockServer, Request, ResponseTemplate};
+ let server = MockServer::start().await;
+ let seen = Arc::new(Mutex::new(Vec::<Value>::new()));
+ let captured = seen.clone();
+ Mock::given(wiremock::matchers::method("POST")).respond_with(move |r: &Request| {
+ let mut seen = captured.lock().unwrap();
+ seen.push(r.body_json().unwrap());
+ if seen.len() == 1 {
+ ResponseTemplate::new(400).set_body_json(json!({"error":{"code":"context_length_exceeded","message":"Prompt is too long"}}))
+ } else {
+ ResponseTemplate::new(200).insert_header("content-type", "text/event-stream")
+ .set_body_string("data: {\"choices\":[{\"delta\":{\"content\":\"Answer from evidence.\"},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n")
+ }
+ }).mount(&server).await;
+ let history = vec![
+ ("user".into(), "old question".into()),
+ ("assistant".into(), "x".repeat(8000)),
+ ];
+ let context =
+ super::super::super::chat_context::Context::new(history.clone(), vec![], 32_000);
+ let evidence = vec![
+ json!({"role":"tool","tool_call_id":"c","content":"[1] immutable evidence","is_error":false}),
+ ];
+ let sources = vec![json!({"n":1,"document_id":"source"})];
+ let input = AnswerContext {
+ question: "current",
+ history: &history,
+ current: None,
+ prior_exchange: &[],
+ exchange: &evidence,
+ sources: &sources,
+ resolved: &[],
+ };
+ let client = LlmClient::new(&server.uri(), None, "test");
+ assert_eq!(
+ answer_with_context(&client, input, &context).await.unwrap(),
+ "Answer from evidence."
+ );
+ let seen = seen.lock().unwrap();
+ assert_eq!(seen.len(), 2);
+ assert_eq!(seen[0]["messages"][0], seen[1]["messages"][0]);
+ let before: Value =
+ serde_json::from_str(seen[0]["messages"][1]["content"].as_str().unwrap()).unwrap();
+ let after: Value =
+ serde_json::from_str(seen[1]["messages"][1]["content"].as_str().unwrap()).unwrap();
+ assert_eq!(after["conversation_context"]["turns"], json!([]));
+ for key in ["question", "evidence", "sources", "resolved_entities"] {
+ assert_eq!(before[key], after[key]);
+ }
+ let err = anyhow::Error::new(utopia_llm::ContextTooLong {
+ status: 400,
+ reason: "400 Bad Request".into(),
+ detail: "again".into(),
+ window: None,
+ });
+ assert!(
+ !context.recover(&client, &err),
+ "a different phase cannot spend another recovery"
+ );
+ }
+
#[test]
fn evidence_and_citations_are_data_not_protocol_messages() {
let exchange = vec![
@@ -239,7 +320,10 @@ mod tests {
resolved: &[],
};
let client = LlmClient::new("http://127.0.0.1:1", None, "test");
- let error = answer(&client, input).await.unwrap_err();
+ let context = super::super::super::chat_context::Context::new(vec![], vec![], 32_000);
+ let error = answer_with_context(&client, input, &context)
+ .await
+ .unwrap_err();
assert!(error.to_string().contains("context limit"));
}
#[tokio::test]
@@ -268,7 +352,7 @@ mod tests {
sources: &[],
resolved: &[],
};
- let err = answer_with_deadline(&client, input, Duration::from_millis(200))
+ let err = answer_with_deadline(&client, input, Duration::from_millis(200), None)
.await
.unwrap_err();
assert!(err.to_string().contains("timed out"));
diff --git a/crates/utopia-server/src/api/mod.rs b/crates/utopia-server/src/api/mod.rs
index bc9b1a08..b75830e1 100644
--- a/crates/utopia-server/src/api/mod.rs
+++ b/crates/utopia-server/src/api/mod.rs
@@ -3,6 +3,7 @@ mod agent;
mod alerts_routes;
mod auth_routes;
mod chat;
+mod chat_context;
mod datasource_routes;
pub(crate) mod documents_routes;
mod events_routes;
diff --git a/crates/utopia-server/src/api/rig_model.rs b/crates/utopia-server/src/api/rig_model.rs
index 4c44ea39..e6ac9bdc 100644
--- a/crates/utopia-server/src/api/rig_model.rs
+++ b/crates/utopia-server/src/api/rig_model.rs
@@ -23,11 +23,12 @@ const PROVIDER: &str = "utopia";
#[derive(Clone)]
pub struct RigModel {
client: LlmClient,
+ context: super::chat_context::Context,
}
impl RigModel {
- pub fn new(client: LlmClient) -> Self {
- Self { client }
+ pub(super) fn with_context(client: LlmClient, context: super::chat_context::Context) -> Self {
+ Self { client, context }
}
/// 开一条流。**端点不接受 `tool_choice` 时去掉它重发一次**:`required` 不在
@@ -38,20 +39,22 @@ impl RigModel {
w: &Wire,
) -> Result<impl Stream<Item = anyhow::Result<ToolStreamItem>> + Send + use<>, CompletionError>
{
- match self
- .client
- .chat_tools_stream_with(&w.messages, w.tools.as_ref(), w.tool_choice.as_ref())
- .await
- {
- Ok(s) => Ok(s),
- Err(e) if w.tool_choice.is_some() && rejected_shape(&e) => {
- tracing::warn!(error = %e, "端点不接受 tool_choice,去掉重发");
- self.client
- .chat_tools_stream_with(&w.messages, w.tools.as_ref(), None)
- .await
- .map_err(completion_error)
+ let mut choice = w.tool_choice.as_ref();
+ loop {
+ let messages = self.context.apply(&w.messages);
+ match self
+ .client
+ .chat_tools_stream_with(&messages, w.tools.as_ref(), choice)
+ .await
+ {
+ Ok(s) => return Ok(s),
+ Err(e) if self.context.recover(&self.client, &e) => continue,
+ Err(e) if choice.is_some() && rejected_shape(&e) => {
+ tracing::warn!(error = %e, "端点不接受 tool_choice,去掉重发");
+ choice = None;
+ }
+ Err(e) => return Err(completion_error(e)),
}
- Err(e) => Err(completion_error(e)),
}
}
}
@@ -62,20 +65,22 @@ impl CompletionModel for RigModel {
request: CompletionRequest,
) -> Result<CompletionResponse, CompletionError> {
let w = wire(&request);
- let turn = match self
- .client
- .chat_tools_with(&w.messages, w.tools.as_ref(), w.tool_choice.as_ref())
- .await
- {
- Ok(t) => t,
- Err(e) if w.tool_choice.is_some() && rejected_shape(&e) => {
- tracing::warn!(error = %e, "端点不接受 tool_choice,去掉重发");
- self.client
- .chat_tools_with(&w.messages, w.tools.as_ref(), None)
- .await
- .map_err(completion_error)?
+ let mut choice = w.tool_choice.as_ref();
+ let turn = loop {
+ let messages = self.context.apply(&w.messages);
+ match self
+ .client
+ .chat_tools_with(&messages, w.tools.as_ref(), choice)
+ .await
+ {
+ Ok(t) => break t,
+ Err(e) if self.context.recover(&self.client, &e) => continue,
+ Err(e) if choice.is_some() && rejected_shape(&e) => {
+ tracing::warn!(error = %e, "端点不接受 tool_choice,去掉重发");
+ choice = None;
+ }
+ Err(e) => return Err(completion_error(e)),
}
- Err(e) => return Err(completion_error(e)),
};
Ok(
CompletionResponse::new(choice_of(&turn), Usage::new(), PROVIDER)
@@ -166,9 +171,12 @@ pub fn llm_failure(err: &CompletionError) -> Option<&anyhow::Error> {
}
/// 端点拒绝了请求的形状(400/422)。带工具的首个请求撞上它,才是「这家不支持
-/// 工具调用」;网断、密钥错、限流、欠费都不是,降级也救不了它们
+/// 工具调用」;网断、密钥错、限流、欠费都不是,降级也救不了它们。上下文超限也是
+/// 400,但那是历史太长不是形状不对:裁掉旧问答重发才是解法,退成 RAG 只会把同样
+/// 长的历史再发一遍(#964)
fn rejected_shape(err: &anyhow::Error) -> bool {
- utopia_llm::rejected(err).is_some_and(|r| r.status == 400 || r.status == 422)
+ utopia_llm::context_too_long(err).is_none()
+ && utopia_llm::rejected(err).is_some_and(|r| r.status == 400 || r.status == 422)
}
/// 见 [`rejected_shape`],从 rig 的错误上判
@@ -259,7 +267,7 @@ fn tool_choice_json(choice: &ToolChoice) -> Option<Value> {
}
}
-fn push_message(out: &mut Vec<Value>, m: &Message) {
+pub(super) fn push_message(out: &mut Vec<Value>, m: &Message) {
match m {
Message::System { content } => out.push(json!({ "role": "system", "content": content })),
Message::User { content } => {
diff --git a/crates/utopia-server/src/llm_util.rs b/crates/utopia-server/src/llm_util.rs
index 5ef16c60..f0bc07e8 100644
--- a/crates/utopia-server/src/llm_util.rs
+++ b/crates/utopia-server/src/llm_util.rs
@@ -23,6 +23,48 @@ pub fn chat_client(s: &LlmSettings) -> Option<LlmClient> {
)
}
+/// 对话模型的设置:地址、密钥、模型、推理力度。任一项变了就是另一个客户端
+type ChatConfig = (
+ Option<String>,
+ Option<String>,
+ Option<String>,
+ Option<String>,
+);
+
+/// 各工作区的对话客户端,跨轮次留着。
+///
+/// 端点报过的上下文窗口记在客户端上(`LlmClient::remember_context_window`),每轮新建一个
+/// 就等于每轮都忘,下一轮照样撞上同一个 400(#964)。模型设置一变就换新的:学到的窗口
+/// 是那个模型的,不是这个工作区的。密钥和窗口都不跨工作区
+#[derive(Default)]
+pub struct ConversationClients {
+ inner: std::sync::Mutex<HashMap<uuid::Uuid, (ChatConfig, LlmClient)>>,
+}
+
+impl ConversationClients {
+ pub fn get(&self, workspace: uuid::Uuid, s: &LlmSettings) -> Option<LlmClient> {
+ let mut clients = self.inner.lock().unwrap();
+ if !s.chat_ready() {
+ clients.remove(&workspace);
+ return None;
+ }
+ let config = (
+ s.chat_base_url.clone(),
+ s.chat_api_key.clone(),
+ s.chat_model.clone(),
+ s.chat_reasoning_effort.clone(),
+ );
+ if let Some((previous, client)) = clients.get(&workspace) {
+ if previous == &config {
+ return Some(client.clone());
+ }
+ }
+ let client = chat_client(s)?;
+ clients.insert(workspace, (config, client.clone()));
+ Some(client)
+ }
+}
+
/// 一轮对话里嵌过的文字与它的向量(#971)。挑口径、检索分块、按名字查实体都要把问题嵌成
/// 向量,一轮之内是同一句话、同一个模型,嵌一次就够。缓存跟着这一轮的 `ToolSink` 走,不落库,
/// 只记成功的;没有这一轮的调用方(HTTP 检索、口径页)传 `None`,各嵌各的
diff --git a/crates/utopia-server/src/state.rs b/crates/utopia-server/src/state.rs
index e1adffdc..b2efbbfc 100644
--- a/crates/utopia-server/src/state.rs
+++ b/crates/utopia-server/src/state.rs
@@ -33,6 +33,8 @@ pub struct AppState {
pub chunk_tokens: usize,
/// worker 并发数:调度循环每轮热读——系统设置改动即时生效
pub worker_concurrency: Arc<std::sync::atomic::AtomicUsize>,
+ /// 各工作区的对话客户端。端点报过的上下文窗口记在客户端上,留着它下一轮才记得(#964)
+ pub chat_clients: Arc<crate::llm_util::ConversationClients>,
/// 按模型的并发闸门:后台任务调 LLM 前取许可。限额存库,改完即时生效
pub model_gates: Arc<crate::llm_util::ModelGates>,
pub events: broadcast::Sender<AppEvent>,
@@ -66,6 +68,7 @@ impl AppState {
// 报错」兜底也是 64——这是**服务起来后第一次读这个值之前**的值,写错了
// 就意味着服务启动那一小会儿跑的是迁移 0011 想避免的并发不足。
worker_concurrency: Arc::new(std::sync::atomic::AtomicUsize::new(64)),
+ chat_clients: Arc::new(crate::llm_util::ConversationClients::default()),
model_gates: Arc::new(crate::llm_util::ModelGates::default()),
events,
live: Arc::new(crate::live::Registry::default()),
diff --git a/docs/decisions/0042-the-chat-loop-is-a-runner-with-hooks.md b/docs/decisions/0042-the-chat-loop-is-a-runner-with-hooks.md
index 3cf1b6eb..fb8f17c8 100644
--- a/docs/decisions/0042-the-chat-loop-is-a-runner-with-hooks.md
+++ b/docs/decisions/0042-the-chat-loop-is-a-runner-with-hooks.md
@@ -1,6 +1,6 @@
# 0042 · The chat loop is a runner with hooks
-- **Status**: Implemented (#548) · revised 2026-09-26 (#937)
+- **Status**: Implemented (#548) · revised 2026-09-26 (#937) · revised 2026-09-27 (#973)
- **Written**: 2026-09-13 (conventions in the [README](README.md))
- **Related**: #546 (the issue and its findings), #509 / #543 (the stall and the guard this
replaces), #547 (the mark on answers that cite nothing), #631 (the empty-reply retry, moved
@@ -47,7 +47,10 @@ earlier entities become a `system` message right before the question, and `ToolC
omits tool fields on the wire.
Degradation to one-shot RAG happens only when the first request that carries tools comes back
-400 or 422 (`utopia_llm::Rejected`). A network failure is an error frame.
+400 or 422 (`utopia_llm::Rejected`). A network failure is an error frame. Revised 2026-09-27
+(#973): a context-window refusal is `ContextTooLong`, not `Rejected`; it drops older history and
+sends the request once more, see
+[below](#a-context-refusal-trims-history-once-973-revised-2026-09-27).
### 3. A turn cannot end before a tool has run
@@ -142,6 +145,35 @@ This boundary is not a factuality oracle. The evaluation separately records requ
slots, citation syntax/mapping, additional unsupported statements and false insufficiency.
A clean result on one frozen set is not a zero-failure guarantee.
+## A context refusal trims history once (#973, revised 2026-09-27)
+
+The 20-message window alone let ten large answers make every later turn fail (#964). The
+endpoint's context refusal was read as unsupported tools, so removing `tool_choice` and then
+falling back to RAG sent the same oversized history twice more. Three things change.
+
+**The refusal has its own class.** `utopia-llm` reads `context_length_exceeded` from the error
+code first, then the vendor's wording (`maximum context length`, `context window`, `too many
+tokens`, `prompt is too long`, `exceeds the available context`), next to
+`stated_completion_ceiling`, and `rejected_shape` is false for it. DeepSeek's 400 carries only the
+generic `invalid_request_error` code; its sentence "This model's maximum context length is 1048576
+tokens. However, you requested …" is what is recognized (confirmed 2026-09-27).
+
+**History is bounded by size as well as by count.** Inside the 20-message window at most 32,000
+characters of history are sent, the oldest whole exchanges dropped first; a failed question and
+a stopped answer count like any other, and an orphan answer at the count boundary is not sent.
+The previous turn's tool exchange is the first thing to go: one `get_document` result can be
+24,000 characters, and tying it to the last answer meant a follow-up such as "make it shorter"
+went out with no history at all. The budget is a character heuristic, not token accounting.
+
+**One recovery per user turn.** On a recognized refusal the remaining history is halved and the
+request sent once more; the current question, the system prompt and the current turn's tool
+results stay whole, and no tool runs again. Tool calling, the RAG fallback and the final
+evidence-only answer share that one retry. When nothing can be dropped there is no retry. A
+second refusal ends the turn with `context_too_long`, worded as the conversation being too long
+for the model. The workspace's conversation client remembers a stated window of at least 1,024
+tokens across turns and clones; the budget becomes half that many characters and never falls
+below 2,000; changing the model settings forgets it. Stored messages are never altered.
+
## Not done
- A per-task model (`on_model_select`, #470) is available in the runner and not wired.
diff --git a/docs/design/chat-and-mcp.md b/docs/design/chat-and-mcp.md
index 0d30c91e..67d49193 100644
--- a/docs/design/chat-and-mcp.md
+++ b/docs/design/chat-and-mcp.md
@@ -20,10 +20,22 @@ shorter"; `STALL_NUDGE` and `DONE` are gone [0042 d3]. The wire stays `LlmClient
error bodies, the classification of a failure as out of credit, rate limited, unavailable or nothing
of the kind, and cache logging; earlier
entities become a `system` message right before the question; degradation to one-shot RAG happens
-only on a 400 or 422 to the first request with tools [0042 d2]. A question whose answer failed stays
+only on a 400 or 422 to the first request with tools that is not a context-window refusal [0042 d2]. A question whose answer failed stays
stored without one; a retry names it (`retry_message_id`) and answers it in place, only while it is
the conversation's last message and no answer is being written there (#936).
+**Long conversations.** The 20-message history window also has a 32,000-character default
+budget. The previous turn's tool exchange is dropped first, then the oldest whole exchanges;
+failed questions and stopped answers count too, and an orphan answer at the count boundary is not
+sent. A context-window refusal halves the remaining history and retries the request once for the
+whole user turn. It keeps the current question, system prompt and current tool results intact,
+does not execute tools again, and does not retry when nothing can be dropped. Tool calling,
+one-shot RAG and the reserved final answer share the same window and retry. A second refusal is
+`context_too_long`, offering a new conversation. The workspace's client remembers a stated
+window of at least 1,024 tokens for later turns: the history budget becomes half that many
+characters, between 2,000 and 32,000 (a heuristic, not token accounting). Changing the model
+settings resets that learned limit. Stored conversation messages are never deleted [0042].
+
**Stop** explicitly cancels pending model and tool work, then saves the published partial answer
once with `stopped: true` before sending `done`. The browser waits for that terminal outcome before
allowing a follow-up; navigation and SSE disconnections let generation continue. A conversation
diff --git a/web/src/chatStream.test.ts b/web/src/chatStream.test.ts
index d2b1fc88..2f89b36a 100644
--- a/web/src/chatStream.test.ts
+++ b/web/src/chatStream.test.ts
@@ -5,6 +5,7 @@ vi.mock("./i18n", () => ({
ask: { streamInterrupted: "Stream interrupted" },
err: {
no_chat_model: "Worded: configure a chat model",
+ context_too_long: "Worded: start a new conversation",
model_out_of_credit: "Worded: the model account cannot pay",
answer_running: "Worded: this conversation already has an answer in progress",
},
@@ -85,6 +86,11 @@ describe("a chat stream that fails", () => {
expect(await replay(text)).toEqual([["error", "Worded: the model account cannot pay"]]);
expect(await replay(text, true)).toEqual([["error", "Worded: the model account cannot pay"]]);
});
+ it("offers a new conversation for context overflow on both live and reattached streams", async () => {
+ const text = failed({ error: "maximum context length is 16384 tokens", code: "context_too_long" });
+ expect(await replay(text)).toEqual([["error", "Worded: start a new conversation"]]);
+ expect(await replay(text, true)).toEqual([["error", "Worded: start a new conversation"]]);
+ });
it("keeps the server's sentence for a code the table does not have", async () => {
expect(await replay(failed({ error: "Some new failure", code: "unlisted_code" })))
.toEqual([["error", "Some new failure"]]);
diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts
index 943c15fa..c809e865 100644
--- a/web/src/i18n/en.ts
+++ b/web/src/i18n/en.ts
@@ -72,6 +72,7 @@ export const en = {
model_unavailable: "The model endpoint is unavailable right now. Try again shortly.",
model_unreachable:
"The model endpoint gave no usable answer. Check its address under Administration → Models.",
+ context_too_long: "This conversation is too long for the model. Start a new conversation.",
model_rejected:
"The model endpoint refused the request. Check the model settings under Administration → Models.",
// 重答(#936)被拒。生成中又来一个新问题(#961)也是 409;两边用同一个 code
diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts
index d067ee77..81caf005 100644
--- a/web/src/i18n/zh.ts
+++ b/web/src/i18n/zh.ts
@@ -69,6 +69,7 @@ export const zh: Strings = {
model_rate_limited: "模型端点在限流,稍等一会儿再问。",
model_unavailable: "模型端点暂时不可用,请稍后再试。",
model_unreachable: "模型端点没有给出可用的回答。去「管理 → 模型」里检查端点地址。",
+ context_too_long: "这场对话超出了模型的上下文长度。请新建一场对话。",
model_rejected: "模型端点拒绝了这次请求。去「管理 → 模型」里检查模型设置。",
answer_running: "这个对话里还有一个回答正在生成,等它写完再试。",
retry_answered: "这个问题已经有回答了,或者后面已经接着问了别的。",