Skip to content

World's first open-source enterprise world model.

Choosing the mappings and searching chunks reuse the question's embedding instead of embedding it again in the same turn (#978)

Maya Kidb5298b4d8784
diff --git a/crates/utopia-server/src/api/chat.rs b/crates/utopia-server/src/api/chat.rs
index 50e67397..a3cb73eb 100644
--- a/crates/utopia-server/src/api/chat.rs
+++ b/crates/utopia-server/src/api/chat.rs
@@ -763,11 +763,15 @@ pub async fn chat(
// 三十条的上限之下量的,一百条口径靠字典序截断就不成立了(#574)。
// 挑口径要调嵌入模型,所以放在生成器里:Stop 丢掉生成器,这个请求跟着取消(0063)。
// 代价是它失败得晚了:从前是什么都没写的 500,现在问题已经存下,失败是一帧 error
+ // 这一轮的嵌入缓存从挑口径起就在:问题在这里嵌一次,随后检索分块、按名字查实体
+ // 都拿同一个向量(#971 的后续)。下面建好 sink 就把它交过去
+ let mut embeddings = crate::llm_util::EmbedCache::default();
let mappings = if mounted_sources.is_empty() {
Vec::new()
} else {
match crate::mapping_index::relevant(
&state, kb_id, workspace_id, &query, crate::mapping_index::DEFINITIONS_IN_PROMPT,
+ Some(&mut embeddings),
).await {
Ok(mappings) => mappings,
Err(error) => {
@@ -849,6 +853,7 @@ pub async fn chat(
settings.chat_model.clone().unwrap_or_default(),
query.clone(),
);
+ shared.sink.lock().await.embeddings = embeddings;
let policy = agent::Policy {
shared: shared.clone(),
max_rounds: MAX_ROUNDS,
@@ -1201,7 +1206,8 @@ fn legacy_rag(
client: utopia_llm::LlmClient,
) -> impl Stream<Item = ProducerEvent> {
async_stream::stream! {
- let chunks = match retrieval::hybrid(&state, kb_id, workspace_id, &query, 8, None).await {
+ // 兜底这一路一轮只走一次,问题自己嵌:它没有这一轮的 sink(#971 的缓存在那上面)
+ let chunks = match retrieval::hybrid(&state, kb_id, workspace_id, &query, 8, None, None).await {
Ok(chunks) => chunks,
Err(error) => {
tracing::warn!(%error, "fallback document retrieval failed");
diff --git a/crates/utopia-server/src/api/chat_embed_once_tests.rs b/crates/utopia-server/src/api/chat_embed_once_tests.rs
index c8972dd7..0c981669 100644
--- a/crates/utopia-server/src/api/chat_embed_once_tests.rs
+++ b/crates/utopia-server/src/api/chat_embed_once_tests.rs
@@ -10,7 +10,10 @@
//! - 工具层(同一个 sink 就是同一轮):find_entities、按名字的 entity_facts 与
//! neighbors 都挑中近的那个,问题只嵌一次;同一个对不上的谓词问两次,那个词只嵌
//! 一次;换一个 sink(下一轮),问题再嵌一次
-//! - 整条对话:模型先 find_entities,再按名字 entity_facts,嵌入服务只收到一次请求
+//! - 整条对话:模型先 search_chunks(检索词就是那句问题),再 find_entities、按名字
+//! entity_facts,嵌入服务只收到一次请求
+//! - 挑口径(`mapping_index::relevant`)、检索分块(`retrieval::hybrid`)和按名字查实体
+//! 共用这一轮的缓存:问题只嵌一次;没有缓存的调用方各嵌各的(#971 的后续)
//!
//! 没有 `UTOPIA_DATABASE_URL` 时跳过而不是失败。自建自拆,绝不碰已有的库。
use super::*;
@@ -218,7 +221,9 @@ async fn a_turn_embeds_its_question_and_each_word_once() -> anyhow::Result<()> {
#[tokio::test]
async fn a_chat_turn_embeds_its_question_once() -> anyhow::Result<()> {
+ // 模型先拿那句问题去检索分块:向量那一路把它嵌了,后面按名字查实体拿的是同一个向量
let Some(f) = fixture(Scripted::new(vec![
+ Reply::Tool("search_chunks", r#"{"query":"Where is the Acme lab?"}"#),
Reply::Tool("find_entities", r#"{"name":"Acme"}"#),
Reply::Tool("entity_facts", r#"{"entity_id":"Acme"}"#),
Reply::Text("The Acme lab is in Harbor."),
@@ -231,7 +236,7 @@ async fn a_chat_turn_embeds_its_question_once() -> anyhow::Result<()> {
let run = async {
let sse = f.ask(QUESTION).await?;
assert!(sse.contains("event: done"), "{sse}");
- assert_eq!(sse.matches("event: step").count(), 2, "{sse}");
+ assert_eq!(sse.matches("event: step").count(), 3, "{sse}");
assert_eq!(embedder.requests(), vec![vec![QUESTION.to_string()]]);
anyhow::Ok(())
}
@@ -240,3 +245,82 @@ async fn a_chat_turn_embeds_its_question_once() -> anyhow::Result<()> {
f.cleanup().await?;
run
}
+
+/// 挑口径、检索分块、按名字查实体三处拿的是同一个向量;换一句检索词才再嵌一次;
+/// 没有缓存的调用方(HTTP 检索、口径页)各嵌各的
+#[tokio::test]
+async fn mappings_search_and_the_graph_tools_share_one_embedding() -> anyhow::Result<()> {
+ let Some(f) = fixture(Scripted::new(vec![])).await? else {
+ return Ok(());
+ };
+ let (embedder, server, acme) = seed(&f).await?;
+ let run = async {
+ let ws: Uuid = sqlx::query_scalar("SELECT workspace_id FROM knowledge_bases WHERE id = $1")
+ .bind(f.kb)
+ .fetch_one(&f.pool)
+ .await?;
+ let mut cache = crate::llm_util::EmbedCache::default();
+
+ // 这个库一条确认口径都没有:挑口径两路都空,但问题已经嵌了一次
+ let mappings =
+ crate::mapping_index::relevant(&f.state, f.kb, ws, QUESTION, 8, Some(&mut cache))
+ .await?;
+ assert!(mappings.is_empty());
+ assert_eq!(embedder.requests(), vec![vec![QUESTION.to_string()]]);
+ assert!(cache.contains_key(QUESTION));
+
+ // 检索分块拿同一个向量:索引里没有块,向量那一路照样跑,只是不再嵌
+ let chunks =
+ crate::retrieval::hybrid(&f.state, f.kb, ws, QUESTION, 8, None, Some(&mut cache))
+ .await?;
+ assert!(chunks.is_empty());
+ assert_eq!(embedder.requests().len(), 1, "search reuses the vector");
+
+ // 按名字查实体:还是它,挑中画像近的那个
+ let ctx = ctx(&f, Some(QUESTION)).await?;
+ let mut turn = ToolSink {
+ embeddings: cache,
+ ..ToolSink::default()
+ };
+ let found = dispatch(&ctx, &mut turn, "find_entities", &json!({"name": "Acme"})).await;
+ assert!(
+ found
+ .text
+ .starts_with(&format!("Best match: {} |", acme.near)),
+ "{}",
+ found.text
+ );
+ assert_eq!(
+ embedder.requests().len(),
+ 1,
+ "the graph lookup reuses it too"
+ );
+
+ // 模型把检索词改了:另一句话,嵌一次,记进同一个缓存
+ let _ = crate::retrieval::hybrid(
+ &f.state,
+ f.kb,
+ ws,
+ "Acme lab address",
+ 8,
+ None,
+ Some(&mut turn.embeddings),
+ )
+ .await?;
+ assert_eq!(
+ embedder.requests().last(),
+ Some(&vec!["Acme lab address".to_string()])
+ );
+ assert!(turn.embeddings.contains_key("Acme lab address"));
+
+ // 没有这一轮的调用方各嵌各的
+ let _ = crate::retrieval::hybrid(&f.state, f.kb, ws, QUESTION, 8, None, None).await?;
+ let _ = crate::mapping_index::relevant(&f.state, f.kb, ws, QUESTION, 8, None).await?;
+ assert_eq!(embedder.requests().len(), 4);
+ anyhow::Ok(())
+ }
+ .await;
+ server.abort();
+ f.cleanup().await?;
+ run
+}
diff --git a/crates/utopia-server/src/api/mapping_routes.rs b/crates/utopia-server/src/api/mapping_routes.rs
index 875e9ca2..8b8757ad 100644
--- a/crates/utopia-server/src/api/mapping_routes.rs
+++ b/crates/utopia-server/src/api/mapping_routes.rs
@@ -175,7 +175,7 @@ pub async fn relevant(
let k =
q.k.unwrap_or(crate::mapping_index::DEFINITIONS_IN_PROMPT)
.clamp(1, 50);
- let items = crate::mapping_index::relevant(&state, kb_id, kb.workspace_id, &q.q, k)
+ let items = crate::mapping_index::relevant(&state, kb_id, kb.workspace_id, &q.q, k, None)
.await
.map_err(AppError::Other)?;
Ok(Json(json!({ "items": items, "k": k })))
diff --git a/crates/utopia-server/src/api/search_routes.rs b/crates/utopia-server/src/api/search_routes.rs
index 906d26e1..0b776059 100644
--- a/crates/utopia-server/src/api/search_routes.rs
+++ b/crates/utopia-server/src/api/search_routes.rs
@@ -36,6 +36,6 @@ pub async fn search(
let top_k = req.top_k.unwrap_or(10).min(50);
let as_of = super::graph_routes::parse_instant("as_of", req.as_of.as_deref())?;
- let chunks = retrieval::hybrid(&state, kb_id, kb.workspace_id, q, top_k, as_of).await?;
+ let chunks = retrieval::hybrid(&state, kb_id, kb.workspace_id, q, top_k, as_of, None).await?;
Ok(Json(json!({ "results": chunks })))
}
diff --git a/crates/utopia-server/src/api/tools.rs b/crates/utopia-server/src/api/tools.rs
index bc13424d..f933bfba 100644
--- a/crates/utopia-server/src/api/tools.rs
+++ b/crates/utopia-server/src/api/tools.rs
@@ -99,8 +99,10 @@ pub struct ToolSink {
pub resolved: Vec<serde_json::Value>,
/// 这一轮嵌过的文字与它的向量,不落库。同名实体每按名字查一次,都拿同一句问题去比
/// 画像;谓词对不上时,同一个词也会再嵌。一轮之内文字与模型都不变,嵌一次就够:
- /// 对话一轮一个 sink,MCP 一次调用一个。只记成功的,失败了下次照样再试
- pub embeddings: std::collections::HashMap<String, Vec<f32>>,
+ /// 对话一轮一个 sink,MCP 一次调用一个。只记成功的,失败了下次照样再试。
+ /// 挑口径(`mapping_index::relevant`)与检索分块(`search_chunks`)也从这里拿问题的
+ /// 向量、往这里记(#971 的后续)
+ pub embeddings: crate::llm_util::EmbedCache,
}
/// 界面上的一步(#942)。
@@ -291,6 +293,7 @@ pub async fn search_chunks(
&q,
SEARCH_TOP_K,
as_of,
+ Some(&mut sink.embeddings),
)
.await
{
diff --git a/crates/utopia-server/src/llm_util.rs b/crates/utopia-server/src/llm_util.rs
index 3e229ee7..5ef16c60 100644
--- a/crates/utopia-server/src/llm_util.rs
+++ b/crates/utopia-server/src/llm_util.rs
@@ -23,6 +23,11 @@ pub fn chat_client(s: &LlmSettings) -> Option<LlmClient> {
)
}
+/// 一轮对话里嵌过的文字与它的向量(#971)。挑口径、检索分块、按名字查实体都要把问题嵌成
+/// 向量,一轮之内是同一句话、同一个模型,嵌一次就够。缓存跟着这一轮的 `ToolSink` 走,不落库,
+/// 只记成功的;没有这一轮的调用方(HTTP 检索、口径页)传 `None`,各嵌各的
+pub type EmbedCache = std::collections::HashMap<String, Vec<f32>>;
+
pub fn embed_client(s: &LlmSettings) -> Option<LlmClient> {
if !s.embed_ready() {
return None;
diff --git a/crates/utopia-server/src/mapping_index.rs b/crates/utopia-server/src/mapping_index.rs
index 9204afd0..8f5fcc23 100644
--- a/crates/utopia-server/src/mapping_index.rs
+++ b/crates/utopia-server/src/mapping_index.rs
@@ -13,6 +13,9 @@
//!
//! 索引是**懒的**:`relevant` 先把没嵌的补上再查。一次问数通常没有要补的;
//! 刚确认了一批才会有,那一次多等一个嵌入调用。谁都不用记得去刷新。
+//!
+//! 问题的向量走这一轮的缓存(`llm_util::EmbedCache`,#971 的后续):对话先在这里嵌一次,
+//! 随后的检索分块和按名字查实体都拿同一个向量,一轮只调一次嵌入端点。
use crate::llm_util;
use crate::state::AppState;
@@ -85,6 +88,7 @@ pub async fn relevant(
workspace_id: Uuid,
query: &str,
k: usize,
+ embeddings: Option<&mut llm_util::EmbedCache>,
) -> anyhow::Result<Vec<ConceptMapping>> {
let query = query.trim();
if query.is_empty() {
@@ -101,14 +105,14 @@ pub async fn relevant(
.map(|id| id.to_string())
.collect();
- let vector: Option<Vec<String>> = match vector_channel(state, kb_id, workspace_id, query).await
- {
- Ok(v) => v,
- Err(e) => {
- tracing::warn!(%kb_id, error = %e, "口径向量检索失败,只用词面");
- None
- }
- };
+ let vector: Option<Vec<String>> =
+ match vector_channel(state, kb_id, workspace_id, query, embeddings).await {
+ Ok(v) => v,
+ Err(e) => {
+ tracing::warn!(%kb_id, error = %e, "口径向量检索失败,只用词面");
+ None
+ }
+ };
let mut lists = vec![lexical];
if let Some(v) = vector {
@@ -123,11 +127,14 @@ pub async fn relevant(
Ok(utopia_store::mappings::by_ids(&state.pool, kb_id, &ids).await?)
}
+/// 向量那一路。缓存里有这句话的向量就不再嵌(也不占嵌入闸门的名额),模型名还是要
+/// 读设置:只比同一个模型嵌出来的口径
async fn vector_channel(
state: &AppState,
kb_id: Uuid,
workspace_id: Uuid,
query: &str,
+ cache: Option<&mut llm_util::EmbedCache>,
) -> anyhow::Result<Option<Vec<String>>> {
let Some(settings) = utopia_store::settings::get(&state.pool, workspace_id).await? else {
return Ok(None);
@@ -138,18 +145,28 @@ async fn vector_channel(
) else {
return Ok(None);
};
- let mut vectors = {
- let _permit = llm_util::acquire_embed(state, &settings).await;
- client.embed(&[query.to_string()]).await?
+ let vector = match cache.as_deref().and_then(|c| c.get(query)).cloned() {
+ Some(v) => v,
+ None => {
+ let mut vectors = {
+ let _permit = llm_util::acquire_embed(state, &settings).await;
+ client.embed(&[query.to_string()]).await?
+ };
+ if vectors.is_empty() {
+ return Ok(None);
+ }
+ let v = vectors.remove(0);
+ if let Some(cache) = cache {
+ cache.insert(query.to_string(), v.clone());
+ }
+ v
+ }
};
- if vectors.is_empty() {
- return Ok(None);
- }
let ids = utopia_store::mappings::vector_search(
&state.pool,
kb_id,
model,
- &vectors.remove(0),
+ &vector,
RECALL_PER_CHANNEL as i64,
)
.await?;
diff --git a/crates/utopia-server/src/retrieval.rs b/crates/utopia-server/src/retrieval.rs
index be51e55b..4f2cbccc 100644
--- a/crates/utopia-server/src/retrieval.rs
+++ b/crates/utopia-server/src/retrieval.rs
@@ -29,6 +29,7 @@ pub async fn hybrid(
query: &str,
top_k: usize,
as_of: Option<chrono::DateTime<chrono::Utc>>,
+ embeddings: Option<&mut llm_util::EmbedCache>,
) -> AppResult<Vec<ChunkView>> {
let bm25 = {
let search = state.search.clone();
@@ -40,7 +41,7 @@ pub async fn hybrid(
// `try_join!` 会把模型故障变成检索报错
let (bm25, vector) = tokio::join!(
bm25,
- vector_channel(state, kb_id, workspace_id, query, as_of)
+ vector_channel(state, kb_id, workspace_id, query, as_of, embeddings)
);
let bm25 = bm25
.map_err(|e| utopia_core::AppError::Other(anyhow::anyhow!("BM25 检索线程退出:{e}")))?
@@ -61,31 +62,45 @@ pub async fn hybrid(
/// 向量那一路。没配嵌入模型、嵌入请求失败、回了空,都是 `Ok(None)`:检索照常,
/// 只剩 BM25——这就是模块头说的静默降级。向量查询本身失败仍是 `Err`:那是库的事,
/// 不是模型的事,吞掉会把坏库伪装成没配模型。
+///
+/// 这一轮已经嵌过这句话(挑口径、按名字查实体都会嵌问题)就直接用那个向量,不再读
+/// 设置、不再调一次嵌入端点(#971 的后续);新嵌成功的记进去给后面的用
async fn vector_channel(
state: &AppState,
kb_id: Uuid,
workspace_id: Uuid,
query: &str,
as_of: Option<chrono::DateTime<chrono::Utc>>,
+ cache: Option<&mut llm_util::EmbedCache>,
) -> AppResult<Option<Vec<String>>> {
- let settings = utopia_store::settings::get(&state.pool, workspace_id).await?;
- let Some(client) = settings.as_ref().and_then(llm_util::embed_client) else {
- return Ok(None);
- };
- let mut embeddings = match client.embed(&[query.to_string()]).await {
- Ok(v) => v,
- Err(e) => {
- tracing::warn!(error = %e, "查询 embedding 失败,降级为纯 BM25");
- return Ok(None);
+ let vector = match cache.as_deref().and_then(|c| c.get(query)).cloned() {
+ Some(v) => v,
+ None => {
+ let settings = utopia_store::settings::get(&state.pool, workspace_id).await?;
+ let Some(client) = settings.as_ref().and_then(llm_util::embed_client) else {
+ return Ok(None);
+ };
+ let mut embeddings = match client.embed(&[query.to_string()]).await {
+ Ok(v) => v,
+ Err(e) => {
+ tracing::warn!(error = %e, "查询 embedding 失败,降级为纯 BM25");
+ return Ok(None);
+ }
+ };
+ if embeddings.is_empty() {
+ return Ok(None);
+ }
+ let v = embeddings.remove(0);
+ if let Some(cache) = cache {
+ cache.insert(query.to_string(), v.clone());
+ }
+ v
}
};
- if embeddings.is_empty() {
- return Ok(None);
- }
let ids = utopia_store::documents::vector_search(
&state.pool,
kb_id,
- &embeddings.remove(0),
+ &vector,
RECALL_PER_CHANNEL as i64,
as_of,
)
diff --git a/crates/utopia-server/src/retrieval_tests.rs b/crates/utopia-server/src/retrieval_tests.rs
index 89d27299..aa85174d 100644
--- a/crates/utopia-server/src/retrieval_tests.rs
+++ b/crates/utopia-server/src/retrieval_tests.rs
@@ -118,7 +118,7 @@ impl Fixture {
}
async fn search(&self, query: &str) -> anyhow::Result<Vec<Uuid>> {
- let chunks = super::hybrid(&self.state, self.kb, self.ws, query, 8, None).await?;
+ let chunks = super::hybrid(&self.state, self.kb, self.ws, query, 8, None, None).await?;
Ok(chunks.into_iter().map(|c| c.id).collect())
}
@@ -238,7 +238,7 @@ async fn historical_search_keeps_live_candidates_before_limiting_results() -> an
.search(&f.kb.to_string(), "apple harvest", 24)?;
assert_eq!(raw.len(), 7);
assert_eq!(raw.last().unwrap().chunk_id, f.b.to_string());
- let current = super::hybrid(&f.state, f.kb, f.ws, "apple harvest", 6, None).await?;
+ let current = super::hybrid(&f.state, f.kb, f.ws, "apple harvest", 6, None, None).await?;
assert_eq!(
current.iter().map(|c| c.id.to_string()).collect::<Vec<_>>(),
raw.iter()
@@ -251,8 +251,8 @@ async fn historical_search_keeps_live_candidates_before_limiting_results() -> an
// The first six hits do not exist in March. They must not consume the
// final limit and discard the one recalled chunk that does exist then.
let at = Some("2026-03-01T00:00:00Z".parse()?);
- let six = super::hybrid(&f.state, f.kb, f.ws, "apple harvest", 6, at).await?;
- let seven = super::hybrid(&f.state, f.kb, f.ws, "apple harvest", 7, at).await?;
+ let six = super::hybrid(&f.state, f.kb, f.ws, "apple harvest", 6, at, None).await?;
+ let seven = super::hybrid(&f.state, f.kb, f.ws, "apple harvest", 7, at, None).await?;
let six: Vec<_> = six.into_iter().map(|c| c.id).collect();
let seven: Vec<_> = seven.into_iter().map(|c| c.id).collect();
let expected = vec![f.b];
diff --git a/docs/design/chat-and-mcp.md b/docs/design/chat-and-mcp.md
index c56467cc..09a7cff8 100644
--- a/docs/design/chat-and-mcp.md
+++ b/docs/design/chat-and-mcp.md
@@ -47,7 +47,8 @@ sources for the citation numbers it repeats (#943).
`relaxed_order` iterative scan, plus full text in embedded Tantivy; both take `as_of`; full text is
"now" only; neither takes `at` [0035, 0019, 0022]. Confirmed mappings and a schema document reach
the prompt through retrieval; a conventions document works today where a rule would be exact
-[0011, 0036].
+[0011, 0036]. A turn embeds its question once: choosing the mappings, `search_chunks` and the
+graph lookups share the turn's embedding cache (#971).
**`remember`** writes a memory document at once; its statements go through open extraction and wait
in `pending_facts`; the assistant says the sentence is recorded and its statements will be shown