World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Keep the previous answer's sources for the citation numbers a restatement repeats (#953)
Maya Kidf747610c154d
diff --git a/crates/utopia-server/src/api/chat.rs b/crates/utopia-server/src/api/chat.rsindex c0644531..958ea085 100644--- a/crates/utopia-server/src/api/chat.rs+++ b/crates/utopia-server/src/api/chat.rs@@ -20,6 +20,7 @@ use rig_core::message::Message;use rig_core::streaming::{StreamedAssistantContent, StreamedUserContent};use serde::Deserialize;use serde_json::json;+use std::collections::BTreeSet;use std::convert::Infallible;use utopia_core::models::{ChunkView, Role};use utopia_core::AppError;@@ -1037,10 +1038,22 @@ pub async fn chat(return;}}- let (sources, resolved) = {+ let (mut sources, resolved) = {let sink = shared.sink.lock().await;(sink.sources.clone(), sink.resolved.clone())};+ // 一个工具都没查的一轮(翻译、说短一点)照抄的是上一条回答的 [n]。沿用那些号+ // 在上一条里的来源,角标才点得开(#943);查过东西的一轮只认自己查到的+ if !shared.finalizing() && steps_acc.is_empty() && sources.is_empty() {+ let previous = history+ .turns+ .iter()+ .rev()+ .find(|(role, _)| role == "assistant")+ .map(|(_, content)| content.as_str())+ .unwrap_or_default();+ sources = carried_sources(&answer_acc, previous, &history.last_sources);+ }let saved = utopia_store::conversations::append_message(&state.pool, conversation_id, "assistant", &answer_acc,&utopia_store::conversations::TurnRecord {@@ -1287,6 +1300,61 @@ fn error_event(code: &str, message: &str) -> Frame {)}+/// 正文里的引用号,与界面画角标的 `citeRe` 同一个形状:`[1]`、`[1][2]`、`[1, 2]`、+/// `[1,2]`——方括号里只有数字与分隔符,分隔符两边可以有空白,0 不是号+fn cited_numbers(text: &str) -> BTreeSet<u64> {+ let mut out = BTreeSet::new();+ for (at, _) in text.match_indices('[') {+ let rest = &text[at + 1..];+ let Some(close) = rest.find(']') else {+ continue;+ };+ let parts: Vec<&str> = rest[..close].split([',', ',']).collect();+ let mut nums = Vec::new();+ for (k, part) in parts.iter().enumerate() {+ let mut digits = *part;+ if k > 0 {+ digits = digits.trim_start();+ }+ if k + 1 < parts.len() {+ digits = digits.trim_end();+ }+ if digits.is_empty() || !digits.bytes().all(|b| b.is_ascii_digit()) {+ nums.clear();+ break;+ }+ match digits.parse::<u64>() {+ Ok(n) => nums.push(n),+ Err(_) => {+ nums.clear();+ break;+ }+ }+ }+ out.extend(nums.into_iter().filter(|n| *n > 0));+ }+ out+}++/// 不查东西的一轮沿用上一条回答的来源(#943)。只在这一轮引用的号**全都**在上一条+/// 回答里引用过时才给,取上一条存下的来源里被引用到的那几条,号不变;否则一条不给——+/// 不去猜这些号原本属于哪一轮+fn carried_sources(+ answer: &str,+ previous_answer: &str,+ previous_sources: &[serde_json::Value],+) -> Vec<serde_json::Value> {+ let cited = cited_numbers(answer);+ if cited.is_empty() || !cited.is_subset(&cited_numbers(previous_answer)) {+ return Vec::new();+ }+ previous_sources+ .iter()+ .filter(|s| s["n"].as_u64().is_some_and(|n| cited.contains(&n)))+ .cloned()+ .collect()+}+fn source_json(n: usize, c: &ChunkView) -> serde_json::Value {json!({"n": n,diff --git a/crates/utopia-server/src/api/chat_empty_reply_tests.rs b/crates/utopia-server/src/api/chat_empty_reply_tests.rsindex 893a73f4..54c719ff 100644--- a/crates/utopia-server/src/api/chat_empty_reply_tests.rs+++ b/crates/utopia-server/src/api/chat_empty_reply_tests.rs@@ -386,6 +386,8 @@ mod persistence_tests;mod registry_tests;#[path = "chat_resolved_tests.rs"]mod resolved_tests;+#[path = "chat_restated_tests.rs"]+mod restated_tests;#[path = "chat_sources_tests.rs"]mod sources_tests;diff --git a/crates/utopia-server/src/api/chat_history_tests.rs b/crates/utopia-server/src/api/chat_history_tests.rsindex 1ae3147c..4fd8bf16 100644--- a/crates/utopia-server/src/api/chat_history_tests.rs+++ b/crates/utopia-server/src/api/chat_history_tests.rs@@ -28,7 +28,7 @@ fn last_user(request: &serde_json::Value) -> String {}/// Ask in an existing conversation and collect the whole SSE body-async fn ask_in(f: &Fx, conversation_id: Uuid, message: &str) -> anyhow::Result<String> {+pub(super) async fn ask_in(f: &Fx, conversation_id: Uuid, message: &str) -> anyhow::Result<String> {let sse = chat(State(f.state.clone()),AuthUser(f.user.clone()),diff --git a/crates/utopia-server/src/api/chat_restated_tests.rs b/crates/utopia-server/src/api/chat_restated_tests.rsnew file mode 100644index 00000000..18e695cc--- /dev/null+++ b/crates/utopia-server/src/api/chat_restated_tests.rs@@ -0,0 +1,172 @@+//! A restatement keeps the sources it repeats (#943).+//!+//! Asked to shorten or translate the previous answer, the assistant answers from the+//! transcript without gathering anything, as the prompt tells it to, and copies the answer's+//! `[n]`. Each turn starts with no sources, so those marks used to open nothing, and the+//! answer was labelled as citing no sources. A turn that gathers nothing now takes the previous+//! answer's sources for the numbers it repeats, and only when every number it cites was cited+//! there; otherwise it keeps none rather than guess which answer a number came from.+use super::history_tests::ask_in;+use super::*;++async fn seed_document(f: &Fx, document: Uuid) -> anyhow::Result<()> {+ sqlx::query(+ "INSERT INTO documents(id,kb_id,filename,sha256) VALUES($1,$2,'target.md',repeat('0',64))",+ )+ .bind(document)+ .bind(f.kb)+ .execute(&f.pool)+ .await?;+ sqlx::query(+ "INSERT INTO chunks(id,kb_id,document_id,seq,text) \+ VALUES($1,$2,$3,0,'The planned target is 95%, not a measured result.')",+ )+ .bind(Uuid::now_v7())+ .bind(f.kb)+ .bind(document)+ .execute(&f.pool)+ .await?;+ Ok(())+}++/// Ask the first question, then the follow-up in the same conversation+async fn two_turns(f: &Fx) -> anyhow::Result<String> {+ let first = f.ask("What is the documented target?").await?;+ assert!(first.contains("event: done"), "{first}");+ let id: Uuid = sqlx::query_scalar("SELECT id FROM conversations WHERE kb_id = $1")+ .bind(f.kb)+ .fetch_one(&f.pool)+ .await?;+ let second = ask_in(f, id, "Say it shorter").await?;+ assert!(second.contains("event: done"), "{second}");+ Ok(second)+}++/// The sources stored with each answer, oldest first+async fn stored_sources(f: &Fx) -> anyhow::Result<Vec<serde_json::Value>> {+ Ok(sqlx::query_scalar(+ "SELECT m.sources FROM conversation_messages m+ JOIN conversations c ON c.id = m.conversation_id+ WHERE c.kb_id = $1 AND m.role = 'assistant'+ ORDER BY m.created_at",+ )+ .bind(f.kb)+ .fetch_all(&f.pool)+ .await?)+}++/// The last `sources` frame the reader was sent+fn live_sources(sse: &str) -> serde_json::Value {+ let data = sse+ .split("\n\n")+ .filter_map(|frame| frame.strip_prefix("event: sources\ndata: "))+ .last()+ .expect("a sources frame");+ serde_json::from_str(data).expect("sources are JSON")+}++const FIRST: Reply = Reply::Text("The documented target is 95% [1].");+const RESTATE: Reply = Reply::Tool("no_evidence_needed", r#"{"reason":"shorter"}"#);++#[tokio::test]+async fn a_restatement_keeps_the_sources_it_repeats() -> anyhow::Result<()> {+ let document = Uuid::now_v7();+ let Some(f) = fixture(Scripted::new(vec![+ Reply::Document(document),+ FIRST,+ RESTATE,+ Reply::Text("Target: 95% [1]."),+ ]))+ .await?+ else {+ return Ok(());+ };+ seed_document(&f, document).await?;++ let second = two_turns(&f).await?;++ let stored = stored_sources(&f).await?;+ assert_eq!(stored.len(), 2);+ assert_eq!(stored[0][0]["n"], 1, "{stored:?}");+ assert_eq!(+ stored[1], stored[0],+ "the restated [1] opens the passage the first answer cited"+ );+ assert_eq!(+ live_sources(&second),+ stored[1],+ "live and after reload alike"+ );+ f.cleanup().await+}++#[tokio::test]+async fn a_restatement_citing_a_number_the_last_answer_did_not_keeps_none() -> anyhow::Result<()> {+ let document = Uuid::now_v7();+ let Some(f) = fixture(Scripted::new(vec![+ Reply::Document(document),+ FIRST,+ RESTATE,+ Reply::Text("Target: 95% [2]."),+ ]))+ .await?+ else {+ return Ok(());+ };+ seed_document(&f, document).await?;++ let second = two_turns(&f).await?;++ let stored = stored_sources(&f).await?;+ assert_eq!(+ stored[1],+ serde_json::json!([]),+ "[2] was never cited: no guessing"+ );+ assert_eq!(live_sources(&second), serde_json::json!([]));+ f.cleanup().await+}++#[tokio::test]+async fn a_turn_that_gathers_keeps_only_its_own_sources() -> anyhow::Result<()> {+ let document = Uuid::now_v7();+ let Some(f) = fixture(Scripted::new(vec![+ Reply::Document(document),+ FIRST,+ Reply::Tool("search_chunks", r#"{"query":"target"}"#),+ Reply::Text("Target: 95% [1]."),+ ]))+ .await?+ else {+ return Ok(());+ };+ seed_document(&f, document).await?;++ let second = two_turns(&f).await?;++ assert!(+ second.contains("event: step"),+ "the second turn searched: {second}"+ );+ let stored = stored_sources(&f).await?;+ assert_eq!(+ stored[1],+ serde_json::json!([]),+ "a turn that searched answers for its own sources"+ );+ f.cleanup().await+}++/// The server reads citation numbers in the shape the page draws as marks (`citeRe`)+#[test]+fn cited_numbers_reads_the_shapes_the_page_marks() {+ let read = |text: &str| cited_numbers(text).into_iter().collect::<Vec<_>>();+ assert_eq!(+ read("见 [1][3] 与 [4, 6],另见 [2,5]"),+ vec![1, 2, 3, 4, 5, 6]+ );+ for text in ["[ 1]", "[1 ]", "rows[0]", "[a1]", "[1,]", "[+1]", "[]"] {+ assert!(read(text).is_empty(), "{text}");+ }+ assert_eq!(read("[[2]]"), vec![2]);+}diff --git a/crates/utopia-store/src/conversations.rs b/crates/utopia-store/src/conversations.rsindex 99ec13b1..c4d069fd 100644--- a/crates/utopia-store/src/conversations.rs+++ b/crates/utopia-store/src/conversations.rs@@ -1,7 +1,6 @@//! Chat 会话持久化:对话/消息仓储。轨迹(steps)与引用(sources)随//! assistant 消息落库,历史回放与实时流共用同一数据形状。-use chrono::{DateTime, Utc};use sqlx::PgPool;use utopia_core::models::{ConversationMessage, ConversationView};use utopia_core::{AppError, AppResult};@@ -151,18 +150,25 @@ pub struct History {/// 另一批同名实体)。搬二十轮的工具输出回来是另一回事,那正是当初/// 只存正文的理由。pub last_tool_exchange: Vec<serde_json::Value>,+ /// 同一条助手消息的来源(#943)。不查东西的改写会照抄那条回答的 `[n]`,+ /// 这一轮拿它们接上,号才点得开+ pub last_sources: Vec<serde_json::Value>,+}++/// `recent_context` 读的一行消息+#[derive(sqlx::FromRow)]+struct ContextRow {+ id: Uuid,+ role: String,+ content: String,+ resolved: serde_json::Value,+ tool_exchange: serde_json::Value,+ sources: serde_json::Value,}pub async fn recent_context(pool: &PgPool, conversation_id: Uuid, n: i64) -> AppResult<History> {- let mut rows: Vec<(- Uuid,- String,- String,- serde_json::Value,- serde_json::Value,- DateTime<Utc>,- )> = sqlx::query_as(- "SELECT id, role, content, resolved, tool_exchange, created_at FROM conversation_messages+ let mut rows: Vec<ContextRow> = sqlx::query_as(+ "SELECT id, role, content, resolved, tool_exchange, sources FROM conversation_messagesWHERE conversation_id = $1 ORDER BY created_at DESC LIMIT $2",).bind(conversation_id)@@ -174,8 +180,8 @@ pub async fn recent_context(pool: &PgPool, conversation_id: Uuid, n: i64) -> App// 每轮各列一遍只是把同一件事说三遍let mut seen: std::collections::HashSet<String> = std::collections::HashSet::new();let mut entities: Vec<serde_json::Value> = Vec::new();- for (_, _, _, res, _, _) in &rows {- for e in res.as_array().into_iter().flatten() {+ for row in &rows {+ for e in row.resolved.as_array().into_iter().flatten() {let Some(id) = e["id"].as_str() else { continue };if seen.insert(id.to_string()) {entities.push(e.clone());@@ -183,17 +189,22 @@ pub async fn recent_context(pool: &PgPool, conversation_id: Uuid, n: i64) -> App}}// 最后一条助手消息的那一段。**倒着找**——最后一条通常是刚落库的用户消息- let last_tool_exchange = rows- .iter()- .rev()- .find(|(_, role, _, _, _, _)| role == "assistant")- .and_then(|(_, _, _, _, ex, _)| ex.as_array().cloned())+ let last_assistant = rows.iter().rev().find(|row| row.role == "assistant");+ let last_tool_exchange = last_assistant+ .and_then(|row| row.tool_exchange.as_array().cloned())+ .unwrap_or_default();+ let last_sources = last_assistant+ .and_then(|row| row.sources.as_array().cloned()).unwrap_or_default();Ok(History {- turn_ids: rows.iter().map(|(id, ..)| *id).collect(),- turns: rows.into_iter().map(|(_, r, c, _, _, _)| (r, c)).collect(),+ turn_ids: rows.iter().map(|row| row.id).collect(),+ turns: rows+ .into_iter()+ .map(|row| (row.role, row.content))+ .collect(),entities,last_tool_exchange,+ last_sources,})}diff --git a/docs/design/chat-and-mcp.md b/docs/design/chat-and-mcp.mdindex 2ae95d49..eba35270 100644--- a/docs/design/chat-and-mcp.md+++ b/docs/design/chat-and-mcp.md@@ -25,7 +25,8 @@ only on a 400 or 422 to the first request with tools [0042 d2].`at` respected), `changes`, `list_rules`, `rule_matches`, `remember`; MCP results carry`structuredContent` with ledger UUIDs and each evidence row's origin [0014, 0021, 0041, 0019,0022, 0040]. The previous turn's tool calls are replayed so the model knows what it did, not only-what it said [0015].+what it said [0015]; a turn that gathers nothing, such as a restatement, keeps the previous answer's+sources for the citation numbers it repeats (#943).**Retrieval.** Hybrid: vector recall on `chunks.embedding` through the per-dimension HNSW index with`relaxed_order` iterative scan, plus full text in embedded Tantivy; both take `as_of`; full text is