Skip to content

World's first open-source enterprise world model.

Keep the previous answer's sources for the citation numbers a restatement repeats (#953)

Maya Kidf747610c154d
diff --git a/crates/utopia-server/src/api/chat.rs b/crates/utopia-server/src/api/chat.rs
index c0644531..958ea085 100644
--- a/crates/utopia-server/src/api/chat.rs
+++ b/crates/utopia-server/src/api/chat.rs
@@ -20,6 +20,7 @@ use rig_core::message::Message;
use rig_core::streaming::{StreamedAssistantContent, StreamedUserContent};
use serde::Deserialize;
use serde_json::json;
+use std::collections::BTreeSet;
use std::convert::Infallible;
use utopia_core::models::{ChunkView, Role};
use utopia_core::AppError;
@@ -1037,10 +1038,22 @@ pub async fn chat(
return;
}
}
- let (sources, resolved) = {
+ let (mut sources, resolved) = {
let sink = shared.sink.lock().await;
(sink.sources.clone(), sink.resolved.clone())
};
+ // 一个工具都没查的一轮(翻译、说短一点)照抄的是上一条回答的 [n]。沿用那些号
+ // 在上一条里的来源,角标才点得开(#943);查过东西的一轮只认自己查到的
+ if !shared.finalizing() && steps_acc.is_empty() && sources.is_empty() {
+ let previous = history
+ .turns
+ .iter()
+ .rev()
+ .find(|(role, _)| role == "assistant")
+ .map(|(_, content)| content.as_str())
+ .unwrap_or_default();
+ sources = carried_sources(&answer_acc, previous, &history.last_sources);
+ }
let saved = utopia_store::conversations::append_message(
&state.pool, conversation_id, "assistant", &answer_acc,
&utopia_store::conversations::TurnRecord {
@@ -1287,6 +1300,61 @@ fn error_event(code: &str, message: &str) -> Frame {
)
}
+/// 正文里的引用号,与界面画角标的 `citeRe` 同一个形状:`[1]`、`[1][2]`、`[1, 2]`、
+/// `[1,2]`——方括号里只有数字与分隔符,分隔符两边可以有空白,0 不是号
+fn cited_numbers(text: &str) -> BTreeSet<u64> {
+ let mut out = BTreeSet::new();
+ for (at, _) in text.match_indices('[') {
+ let rest = &text[at + 1..];
+ let Some(close) = rest.find(']') else {
+ continue;
+ };
+ let parts: Vec<&str> = rest[..close].split([',', ',']).collect();
+ let mut nums = Vec::new();
+ for (k, part) in parts.iter().enumerate() {
+ let mut digits = *part;
+ if k > 0 {
+ digits = digits.trim_start();
+ }
+ if k + 1 < parts.len() {
+ digits = digits.trim_end();
+ }
+ if digits.is_empty() || !digits.bytes().all(|b| b.is_ascii_digit()) {
+ nums.clear();
+ break;
+ }
+ match digits.parse::<u64>() {
+ Ok(n) => nums.push(n),
+ Err(_) => {
+ nums.clear();
+ break;
+ }
+ }
+ }
+ out.extend(nums.into_iter().filter(|n| *n > 0));
+ }
+ out
+}
+
+/// 不查东西的一轮沿用上一条回答的来源(#943)。只在这一轮引用的号**全都**在上一条
+/// 回答里引用过时才给,取上一条存下的来源里被引用到的那几条,号不变;否则一条不给——
+/// 不去猜这些号原本属于哪一轮
+fn carried_sources(
+ answer: &str,
+ previous_answer: &str,
+ previous_sources: &[serde_json::Value],
+) -> Vec<serde_json::Value> {
+ let cited = cited_numbers(answer);
+ if cited.is_empty() || !cited.is_subset(&cited_numbers(previous_answer)) {
+ return Vec::new();
+ }
+ previous_sources
+ .iter()
+ .filter(|s| s["n"].as_u64().is_some_and(|n| cited.contains(&n)))
+ .cloned()
+ .collect()
+}
+
fn source_json(n: usize, c: &ChunkView) -> serde_json::Value {
json!({
"n": n,
diff --git a/crates/utopia-server/src/api/chat_empty_reply_tests.rs b/crates/utopia-server/src/api/chat_empty_reply_tests.rs
index 893a73f4..54c719ff 100644
--- a/crates/utopia-server/src/api/chat_empty_reply_tests.rs
+++ b/crates/utopia-server/src/api/chat_empty_reply_tests.rs
@@ -386,6 +386,8 @@ mod persistence_tests;
mod registry_tests;
#[path = "chat_resolved_tests.rs"]
mod resolved_tests;
+#[path = "chat_restated_tests.rs"]
+mod restated_tests;
#[path = "chat_sources_tests.rs"]
mod sources_tests;
diff --git a/crates/utopia-server/src/api/chat_history_tests.rs b/crates/utopia-server/src/api/chat_history_tests.rs
index 1ae3147c..4fd8bf16 100644
--- a/crates/utopia-server/src/api/chat_history_tests.rs
+++ b/crates/utopia-server/src/api/chat_history_tests.rs
@@ -28,7 +28,7 @@ fn last_user(request: &serde_json::Value) -> String {
}
/// Ask in an existing conversation and collect the whole SSE body
-async fn ask_in(f: &Fx, conversation_id: Uuid, message: &str) -> anyhow::Result<String> {
+pub(super) async fn ask_in(f: &Fx, conversation_id: Uuid, message: &str) -> anyhow::Result<String> {
let sse = chat(
State(f.state.clone()),
AuthUser(f.user.clone()),
diff --git a/crates/utopia-server/src/api/chat_restated_tests.rs b/crates/utopia-server/src/api/chat_restated_tests.rs
new file mode 100644
index 00000000..18e695cc
--- /dev/null
+++ b/crates/utopia-server/src/api/chat_restated_tests.rs
@@ -0,0 +1,172 @@
+//! A restatement keeps the sources it repeats (#943).
+//!
+//! Asked to shorten or translate the previous answer, the assistant answers from the
+//! transcript without gathering anything, as the prompt tells it to, and copies the answer's
+//! `[n]`. Each turn starts with no sources, so those marks used to open nothing, and the
+//! answer was labelled as citing no sources. A turn that gathers nothing now takes the previous
+//! answer's sources for the numbers it repeats, and only when every number it cites was cited
+//! there; otherwise it keeps none rather than guess which answer a number came from.
+use super::history_tests::ask_in;
+use super::*;
+
+async fn seed_document(f: &Fx, document: Uuid) -> anyhow::Result<()> {
+ sqlx::query(
+ "INSERT INTO documents(id,kb_id,filename,sha256) VALUES($1,$2,'target.md',repeat('0',64))",
+ )
+ .bind(document)
+ .bind(f.kb)
+ .execute(&f.pool)
+ .await?;
+ sqlx::query(
+ "INSERT INTO chunks(id,kb_id,document_id,seq,text) \
+ VALUES($1,$2,$3,0,'The planned target is 95%, not a measured result.')",
+ )
+ .bind(Uuid::now_v7())
+ .bind(f.kb)
+ .bind(document)
+ .execute(&f.pool)
+ .await?;
+ Ok(())
+}
+
+/// Ask the first question, then the follow-up in the same conversation
+async fn two_turns(f: &Fx) -> anyhow::Result<String> {
+ let first = f.ask("What is the documented target?").await?;
+ assert!(first.contains("event: done"), "{first}");
+ let id: Uuid = sqlx::query_scalar("SELECT id FROM conversations WHERE kb_id = $1")
+ .bind(f.kb)
+ .fetch_one(&f.pool)
+ .await?;
+ let second = ask_in(f, id, "Say it shorter").await?;
+ assert!(second.contains("event: done"), "{second}");
+ Ok(second)
+}
+
+/// The sources stored with each answer, oldest first
+async fn stored_sources(f: &Fx) -> anyhow::Result<Vec<serde_json::Value>> {
+ Ok(sqlx::query_scalar(
+ "SELECT m.sources FROM conversation_messages m
+ JOIN conversations c ON c.id = m.conversation_id
+ WHERE c.kb_id = $1 AND m.role = 'assistant'
+ ORDER BY m.created_at",
+ )
+ .bind(f.kb)
+ .fetch_all(&f.pool)
+ .await?)
+}
+
+/// The last `sources` frame the reader was sent
+fn live_sources(sse: &str) -> serde_json::Value {
+ let data = sse
+ .split("\n\n")
+ .filter_map(|frame| frame.strip_prefix("event: sources\ndata: "))
+ .last()
+ .expect("a sources frame");
+ serde_json::from_str(data).expect("sources are JSON")
+}
+
+const FIRST: Reply = Reply::Text("The documented target is 95% [1].");
+const RESTATE: Reply = Reply::Tool("no_evidence_needed", r#"{"reason":"shorter"}"#);
+
+#[tokio::test]
+async fn a_restatement_keeps_the_sources_it_repeats() -> anyhow::Result<()> {
+ let document = Uuid::now_v7();
+ let Some(f) = fixture(Scripted::new(vec![
+ Reply::Document(document),
+ FIRST,
+ RESTATE,
+ Reply::Text("Target: 95% [1]."),
+ ]))
+ .await?
+ else {
+ return Ok(());
+ };
+ seed_document(&f, document).await?;
+
+ let second = two_turns(&f).await?;
+
+ let stored = stored_sources(&f).await?;
+ assert_eq!(stored.len(), 2);
+ assert_eq!(stored[0][0]["n"], 1, "{stored:?}");
+ assert_eq!(
+ stored[1], stored[0],
+ "the restated [1] opens the passage the first answer cited"
+ );
+ assert_eq!(
+ live_sources(&second),
+ stored[1],
+ "live and after reload alike"
+ );
+ f.cleanup().await
+}
+
+#[tokio::test]
+async fn a_restatement_citing_a_number_the_last_answer_did_not_keeps_none() -> anyhow::Result<()> {
+ let document = Uuid::now_v7();
+ let Some(f) = fixture(Scripted::new(vec![
+ Reply::Document(document),
+ FIRST,
+ RESTATE,
+ Reply::Text("Target: 95% [2]."),
+ ]))
+ .await?
+ else {
+ return Ok(());
+ };
+ seed_document(&f, document).await?;
+
+ let second = two_turns(&f).await?;
+
+ let stored = stored_sources(&f).await?;
+ assert_eq!(
+ stored[1],
+ serde_json::json!([]),
+ "[2] was never cited: no guessing"
+ );
+ assert_eq!(live_sources(&second), serde_json::json!([]));
+ f.cleanup().await
+}
+
+#[tokio::test]
+async fn a_turn_that_gathers_keeps_only_its_own_sources() -> anyhow::Result<()> {
+ let document = Uuid::now_v7();
+ let Some(f) = fixture(Scripted::new(vec![
+ Reply::Document(document),
+ FIRST,
+ Reply::Tool("search_chunks", r#"{"query":"target"}"#),
+ Reply::Text("Target: 95% [1]."),
+ ]))
+ .await?
+ else {
+ return Ok(());
+ };
+ seed_document(&f, document).await?;
+
+ let second = two_turns(&f).await?;
+
+ assert!(
+ second.contains("event: step"),
+ "the second turn searched: {second}"
+ );
+ let stored = stored_sources(&f).await?;
+ assert_eq!(
+ stored[1],
+ serde_json::json!([]),
+ "a turn that searched answers for its own sources"
+ );
+ f.cleanup().await
+}
+
+/// The server reads citation numbers in the shape the page draws as marks (`citeRe`)
+#[test]
+fn cited_numbers_reads_the_shapes_the_page_marks() {
+ let read = |text: &str| cited_numbers(text).into_iter().collect::<Vec<_>>();
+ assert_eq!(
+ read("见 [1][3] 与 [4, 6],另见 [2,5]"),
+ vec![1, 2, 3, 4, 5, 6]
+ );
+ for text in ["[ 1]", "[1 ]", "rows[0]", "[a1]", "[1,]", "[+1]", "[]"] {
+ assert!(read(text).is_empty(), "{text}");
+ }
+ assert_eq!(read("[[2]]"), vec![2]);
+}
diff --git a/crates/utopia-store/src/conversations.rs b/crates/utopia-store/src/conversations.rs
index 99ec13b1..c4d069fd 100644
--- a/crates/utopia-store/src/conversations.rs
+++ b/crates/utopia-store/src/conversations.rs
@@ -1,7 +1,6 @@
//! Chat 会话持久化:对话/消息仓储。轨迹(steps)与引用(sources)随
//! assistant 消息落库,历史回放与实时流共用同一数据形状。
-use chrono::{DateTime, Utc};
use sqlx::PgPool;
use utopia_core::models::{ConversationMessage, ConversationView};
use utopia_core::{AppError, AppResult};
@@ -151,18 +150,25 @@ pub struct History {
/// 另一批同名实体)。搬二十轮的工具输出回来是另一回事,那正是当初
/// 只存正文的理由。
pub last_tool_exchange: Vec<serde_json::Value>,
+ /// 同一条助手消息的来源(#943)。不查东西的改写会照抄那条回答的 `[n]`,
+ /// 这一轮拿它们接上,号才点得开
+ pub last_sources: Vec<serde_json::Value>,
+}
+
+/// `recent_context` 读的一行消息
+#[derive(sqlx::FromRow)]
+struct ContextRow {
+ id: Uuid,
+ role: String,
+ content: String,
+ resolved: serde_json::Value,
+ tool_exchange: serde_json::Value,
+ sources: serde_json::Value,
}
pub async fn recent_context(pool: &PgPool, conversation_id: Uuid, n: i64) -> AppResult<History> {
- let mut rows: Vec<(
- Uuid,
- String,
- String,
- serde_json::Value,
- serde_json::Value,
- DateTime<Utc>,
- )> = sqlx::query_as(
- "SELECT id, role, content, resolved, tool_exchange, created_at FROM conversation_messages
+ let mut rows: Vec<ContextRow> = sqlx::query_as(
+ "SELECT id, role, content, resolved, tool_exchange, sources FROM conversation_messages
WHERE conversation_id = $1 ORDER BY created_at DESC LIMIT $2",
)
.bind(conversation_id)
@@ -174,8 +180,8 @@ pub async fn recent_context(pool: &PgPool, conversation_id: Uuid, n: i64) -> App
// 每轮各列一遍只是把同一件事说三遍
let mut seen: std::collections::HashSet<String> = std::collections::HashSet::new();
let mut entities: Vec<serde_json::Value> = Vec::new();
- for (_, _, _, res, _, _) in &rows {
- for e in res.as_array().into_iter().flatten() {
+ for row in &rows {
+ for e in row.resolved.as_array().into_iter().flatten() {
let Some(id) = e["id"].as_str() else { continue };
if seen.insert(id.to_string()) {
entities.push(e.clone());
@@ -183,17 +189,22 @@ pub async fn recent_context(pool: &PgPool, conversation_id: Uuid, n: i64) -> App
}
}
// 最后一条助手消息的那一段。**倒着找**——最后一条通常是刚落库的用户消息
- let last_tool_exchange = rows
- .iter()
- .rev()
- .find(|(_, role, _, _, _, _)| role == "assistant")
- .and_then(|(_, _, _, _, ex, _)| ex.as_array().cloned())
+ let last_assistant = rows.iter().rev().find(|row| row.role == "assistant");
+ let last_tool_exchange = last_assistant
+ .and_then(|row| row.tool_exchange.as_array().cloned())
+ .unwrap_or_default();
+ let last_sources = last_assistant
+ .and_then(|row| row.sources.as_array().cloned())
.unwrap_or_default();
Ok(History {
- turn_ids: rows.iter().map(|(id, ..)| *id).collect(),
- turns: rows.into_iter().map(|(_, r, c, _, _, _)| (r, c)).collect(),
+ turn_ids: rows.iter().map(|row| row.id).collect(),
+ turns: rows
+ .into_iter()
+ .map(|row| (row.role, row.content))
+ .collect(),
entities,
last_tool_exchange,
+ last_sources,
})
}
diff --git a/docs/design/chat-and-mcp.md b/docs/design/chat-and-mcp.md
index 2ae95d49..eba35270 100644
--- a/docs/design/chat-and-mcp.md
+++ b/docs/design/chat-and-mcp.md
@@ -25,7 +25,8 @@ only on a 400 or 422 to the first request with tools [0042 d2].
`at` respected), `changes`, `list_rules`, `rule_matches`, `remember`; MCP results carry
`structuredContent` with ledger UUIDs and each evidence row's origin [0014, 0021, 0041, 0019,
0022, 0040]. The previous turn's tool calls are replayed so the model knows what it did, not only
-what it said [0015].
+what it said [0015]; a turn that gathers nothing, such as a restatement, keeps the previous answer's
+sources for the citation numbers it repeats (#943).
**Retrieval.** Hybrid: vector recall on `chunks.embedding` through the per-dimension HNSW index with
`relaxed_order` iterative scan, plus full text in embedded Tantivy; both take `as_of`; full text is