Skip to content

World's first open-source enterprise world model.

Retrieval reads the base as it was (#338)

WaylandYang1dd917bc50fd
diff --git a/crates/utopia-server/src/api/chat.rs b/crates/utopia-server/src/api/chat.rs
index bf0859d0..a2bf589b 100644
--- a/crates/utopia-server/src/api/chat.rs
+++ b/crates/utopia-server/src/api/chat.rs
@@ -622,7 +622,7 @@ pub async fn chat(
// 模型可能不支持 tool-calling:降级为一次性 RAG 注入
tracing::warn!(error = %e, "tool-calling 不可用,降级为一次性 RAG");
let chunks =
- retrieval::hybrid(&state, kb_id, workspace_id, &query, 8)
+ retrieval::hybrid(&state, kb_id, workspace_id, &query, 8, None)
.await
.unwrap_or_default();
let legacy_sources: Vec<serde_json::Value> = chunks
diff --git a/crates/utopia-server/src/api/graph_routes.rs b/crates/utopia-server/src/api/graph_routes.rs
index 9fce3a88..63565780 100644
--- a/crates/utopia-server/src/api/graph_routes.rs
+++ b/crates/utopia-server/src/api/graph_routes.rs
@@ -24,7 +24,7 @@ pub(super) async fn require_kb(
/// 两根轴共用这一个解析:`at` 走世界轴(那时世界是什么样),`as_of` 走记录轴
/// (那时我们以为世界是什么样)。**两个参数一路分开**(0019):合成一个控件,
/// 答出来的是另一个问题,而屏幕上看不出来
-fn parse_instant(
+pub(super) fn parse_instant(
field: &str,
raw: Option<&str>,
) -> Result<Option<chrono::DateTime<chrono::Utc>>, AppError> {
diff --git a/crates/utopia-server/src/api/search_routes.rs b/crates/utopia-server/src/api/search_routes.rs
index 2f3fc77a..906d26e1 100644
--- a/crates/utopia-server/src/api/search_routes.rs
+++ b/crates/utopia-server/src/api/search_routes.rs
@@ -16,6 +16,10 @@ pub struct SearchReq {
pub q: String,
#[serde(default)]
pub top_k: Option<usize>,
+ /// 记录轴(0019):按**那一刻库里有的东西**检索。YYYY-MM-DD 或 RFC3339。
+ /// 命中的是当时活着的块、当时还没被删的文档
+ #[serde(default)]
+ pub as_of: Option<String>,
}
pub async fn search(
@@ -31,6 +35,7 @@ pub async fn search(
let kb = utopia_store::access::require_kb(&state.pool, &user, kb_id, Role::Viewer).await?;
let top_k = req.top_k.unwrap_or(10).min(50);
- let chunks = retrieval::hybrid(&state, kb_id, kb.workspace_id, q, top_k).await?;
+ let as_of = super::graph_routes::parse_instant("as_of", req.as_of.as_deref())?;
+ let chunks = retrieval::hybrid(&state, kb_id, kb.workspace_id, q, top_k, as_of).await?;
Ok(Json(json!({ "results": chunks })))
}
diff --git a/crates/utopia-server/src/api/tools.rs b/crates/utopia-server/src/api/tools.rs
index b16a4971..6da762d7 100644
--- a/crates/utopia-server/src/api/tools.rs
+++ b/crates/utopia-server/src/api/tools.rs
@@ -106,9 +106,16 @@ pub async fn search_chunks(
// 必填参数由 `chat::check_call` 在派发之前挡下,所以这里不再回落到
// 用户那句原话——回落产出的是一个看起来没问题的错误答案
let q = args["query"].as_str().unwrap_or_default().to_string();
- let chunks = retrieval::hybrid(ctx.state, ctx.kb_id, ctx.workspace_id, &q, SEARCH_TOP_K)
- .await
- .unwrap_or_default();
+ let chunks = retrieval::hybrid(
+ ctx.state,
+ ctx.kb_id,
+ ctx.workspace_id,
+ &q,
+ SEARCH_TOP_K,
+ None,
+ )
+ .await
+ .unwrap_or_default();
let mut lines = Vec::new();
for c in &chunks {
let n = cite(sink, c.id.to_string(), |n| source_json(n, c));
diff --git a/crates/utopia-server/src/retrieval.rs b/crates/utopia-server/src/retrieval.rs
index 16c757ba..f53c8d0f 100644
--- a/crates/utopia-server/src/retrieval.rs
+++ b/crates/utopia-server/src/retrieval.rs
@@ -1,5 +1,11 @@
//! 混合检索:BM25(Tantivy)+ 向量(pgvector)→ RRF 融合。
//! embedding 未配置或请求失败时静默降级为纯 BM25。
+//!
+//! **`as_of` 只在向量与取块这两处是完整的**(0019 开放问题②)。Tantivy 的索引
+//! 只有"现在"一个版本:重解析会把文档的块整批换掉,旧版本不在索引里。所以
+//! 带时刻检索时,BM25 找回来的东西**是对的**(随后按当时的活性过滤),但它
+//! 找不回当时有、如今已被顶掉的那些块——召回缺一角,命中不会出错。
+//! 要补那一角得给全文索引也建版本,那是另一件事,不在这一刀里。
use crate::llm_util;
use crate::state::AppState;
@@ -15,6 +21,7 @@ pub async fn hybrid(
workspace_id: Uuid,
query: &str,
top_k: usize,
+ as_of: Option<chrono::DateTime<chrono::Utc>>,
) -> AppResult<Vec<ChunkView>> {
let mut lists: Vec<Vec<String>> = Vec::new();
@@ -35,6 +42,7 @@ pub async fn hybrid(
kb_id,
&embeddings.remove(0),
RECALL_PER_CHANNEL as i64,
+ as_of,
)
.await?;
lists.push(ids.into_iter().map(|id| id.to_string()).collect());
@@ -46,5 +54,5 @@ pub async fn hybrid(
let fused = utopia_search::rrf_fuse(&lists, top_k);
let ids: Vec<Uuid> = fused.iter().filter_map(|s| s.parse().ok()).collect();
- utopia_store::documents::chunks_by_ids(&state.pool, kb_id, &ids).await
+ utopia_store::documents::chunks_by_ids(&state.pool, kb_id, &ids, as_of).await
}
diff --git a/crates/utopia-store/src/documents.rs b/crates/utopia-store/src/documents.rs
index 54a93958..677d697a 100644
--- a/crates/utopia-store/src/documents.rs
+++ b/crates/utopia-store/src/documents.rs
@@ -1,3 +1,4 @@
+use chrono::{DateTime, Utc};
use pgvector::Vector;
use sqlx::PgPool;
use utopia_core::models::{ChunkView, Document, DocumentPage};
@@ -721,9 +722,14 @@ pub async fn replace_chunks(
}
}
- // 第二阶段:落选旧块(新版里没有同款文本)→ 软删
+ // 第二阶段:落选旧块(新版里没有同款文本)→ 软删。
+ //
+ // **向量留着**(0019 开放问题②)。从前这里顺手 `embedding = NULL` 省存储,
+ // 而省掉的是两者里更小的那一半:块的**正文**本来就留着(版本回放要它),
+ // 一条 1024 维向量不过几 KB。代价却是历史整段搜不出来——记录轴倒得回去,
+ // 检索却只认现在,as-of 检索一开口就是空的。
sqlx::query(
- "UPDATE chunks SET superseded_at = now(), embedding = NULL
+ "UPDATE chunks SET superseded_at = now()
WHERE document_id = $1 AND superseded_at IS NULL AND NOT (id = ANY($2))",
)
.bind(document_id)
@@ -891,33 +897,47 @@ pub async fn vector_search(
kb_id: Uuid,
embedding: &[f32],
limit: i64,
+ as_of: Option<DateTime<Utc>>,
) -> AppResult<Vec<Uuid>> {
let query_vec = Vector::from(embedding.to_vec());
- let rows: Vec<(Uuid,)> = sqlx::query_as(
- "SELECT id FROM chunks
- WHERE kb_id = $1 AND embedding IS NOT NULL AND superseded_at IS NULL
- AND vector_dims(embedding) = vector_dims($2)
- ORDER BY embedding <=> $2
+ let rows: Vec<(Uuid,)> = sqlx::query_as(&format!(
+ "SELECT id FROM chunks c
+ WHERE c.kb_id = $1 AND c.embedding IS NOT NULL AND {live}
+ AND vector_dims(c.embedding) = vector_dims($2)
+ ORDER BY c.embedding <=> $2
LIMIT $3",
- )
+ live = crate::record_axis::chunk_live_at("c", 4),
+ ))
.bind(kb_id)
.bind(&query_vec)
.bind(limit)
+ .bind(as_of)
.fetch_all(pool)
.await?;
Ok(rows.into_iter().map(|(id,)| id).collect())
}
/// 按 id 集合取分块(带文档名),保持传入顺序。
-pub async fn chunks_by_ids(pool: &PgPool, kb_id: Uuid, ids: &[Uuid]) -> AppResult<Vec<ChunkView>> {
- let rows: Vec<ChunkView> = sqlx::query_as(
+///
+/// `as_of`:记录轴(0019)——那一刻还活着的块、还没被删的文档。文档的删除
+/// 同样是记录轴上的事件(#268 留了墓碑),所以两个条件一起倒。
+pub async fn chunks_by_ids(
+ pool: &PgPool,
+ kb_id: Uuid,
+ ids: &[Uuid],
+ as_of: Option<DateTime<Utc>>,
+) -> AppResult<Vec<ChunkView>> {
+ let rows: Vec<ChunkView> = sqlx::query_as(&format!(
"SELECT c.id, c.document_id, c.seq, c.text, d.filename
FROM chunks c JOIN documents d ON d.id = c.document_id
WHERE c.kb_id = $1 AND c.id = ANY($2)
- AND c.superseded_at IS NULL AND d.deleted_at IS NULL",
- )
+ AND {live} AND {doc_live}",
+ live = crate::record_axis::chunk_live_at("c", 3),
+ doc_live = crate::record_axis::document_live_at("d", 3),
+ ))
.bind(kb_id)
.bind(ids)
+ .bind(as_of)
.fetch_all(pool)
.await?;
// 恢复 RRF 排名顺序
diff --git a/crates/utopia-store/src/record_axis.rs b/crates/utopia-store/src/record_axis.rs
index 6c72ed00..98b72e1c 100644
--- a/crates/utopia-store/src/record_axis.rs
+++ b/crates/utopia-store/src/record_axis.rs
@@ -54,6 +54,15 @@ pub fn conflict_open_at(alias: &str, param: usize) -> String {
)
}
+/// `documents`:文档在 T 时刻还在库里。删除留墓碑(#268),所以"删掉的文档"
+/// 在删除之前的任何时刻都该照常出现——它的分块当时确实是可检索的。
+pub fn document_live_at(alias: &str, param: usize) -> String {
+ format!(
+ "{alias}.created_at <= coalesce(${param}, now()) \
+ AND ({alias}.deleted_at IS NULL OR {alias}.deleted_at > coalesce(${param}, now()))"
+ )
+}
+
/// `chunks`:分块在 T 时刻还是现行版本。证据是否"已消失"要按当时的版本判——
/// 今天被重解析顶掉的段落,在三月的图上仍然是活证据。
pub fn chunk_live_at(alias: &str, param: usize) -> String {
diff --git a/crates/utopia-store/tests/a_search_reads_the_base_as_it_was.rs b/crates/utopia-store/tests/a_search_reads_the_base_as_it_was.rs
new file mode 100644
index 00000000..8dfc5679
--- /dev/null
+++ b/crates/utopia-store/tests/a_search_reads_the_base_as_it_was.rs
@@ -0,0 +1,217 @@
+//! 检索也倒得回去(0019 开放问题②),打在真库上。
+//!
+//! 两件事在这里被钉住:
+//!
+//! 1. **重解析不再丢掉旧块的向量。** 从前落选的块 `embedding = NULL`,省掉的是
+//! 两者里更小的一半——正文本来就留着——而代价是历史整段搜不出来。
+//! 2. **带时刻的检索按当时的活性走**:当时活着的块、当时还没被删的文档。
+//! 删除留墓碑(#268),所以"上周删掉的文档"在上上周的检索里照常命中。
+//!
+//! 全文那一路不在这里:Tantivy 的索引只有"现在"一个版本,带时刻检索时它
+//! 找回来的是对的、但找不全(模块头里写了)。这个测试打的是向量与取块两路。
+
+use pgvector::Vector;
+use sqlx::PgPool;
+use utopia_ingest::ChunkPiece;
+use uuid::Uuid;
+
+fn t(s: &str) -> chrono::DateTime<chrono::Utc> {
+ s.parse().unwrap()
+}
+
+struct Fixture {
+ org: Uuid,
+ kb: Uuid,
+ /// 五月被重解析顶掉的旧块
+ old_chunk: Uuid,
+ /// 顶替它的新块
+ new_chunk: Uuid,
+ /// 属于一篇六月被删掉的文档
+ deleted_doc_chunk: Uuid,
+}
+
+async fn seed(pool: &PgPool) -> anyhow::Result<Fixture> {
+ let (org, ws, kb) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7());
+ let (doc, deleted_doc) = (Uuid::now_v7(), Uuid::now_v7());
+ let (old_chunk, deleted_doc_chunk) = (Uuid::now_v7(), Uuid::now_v7());
+ let tag = Uuid::now_v7();
+
+ sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, 'as-of-search-test')")
+ .bind(org)
+ .execute(pool)
+ .await?;
+ sqlx::query("INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, 'as-of-search-test')")
+ .bind(ws)
+ .bind(org)
+ .execute(pool)
+ .await?;
+ sqlx::query(
+ "INSERT INTO knowledge_bases (id, workspace_id, name)
+ VALUES ($1, $2, 'as-of-search-test')",
+ )
+ .bind(kb)
+ .bind(ws)
+ .execute(pool)
+ .await?;
+ for (id, name, deleted) in [
+ (doc, "handbook.md", None),
+ (deleted_doc, "retired.md", Some("2026-06-01T00:00:00Z")),
+ ] {
+ sqlx::query(
+ "INSERT INTO documents (id, kb_id, filename, sha256, status, created_at, deleted_at)
+ VALUES ($1, $2, $3, $4, 'ready', $5, $6)",
+ )
+ .bind(id)
+ .bind(kb)
+ .bind(name)
+ .bind(format!("sha-{tag}-{name}"))
+ .bind(t("2026-02-01T00:00:00Z"))
+ .bind(deleted.map(t))
+ .execute(pool)
+ .await?;
+ }
+
+ let embed = |v: [f32; 3]| Vector::from(v.to_vec());
+ for (id, document, text, vector) in [
+ (
+ old_chunk,
+ doc,
+ "The 2025 handbook says the office is in Shanghai.",
+ embed([1.0, 0.0, 0.0]),
+ ),
+ (
+ deleted_doc_chunk,
+ deleted_doc,
+ "A retired note about the Shanghai office.",
+ embed([0.9, 0.1, 0.0]),
+ ),
+ ] {
+ sqlx::query(
+ "INSERT INTO chunks (id, kb_id, document_id, seq, text, embedding, created_at)
+ VALUES ($1, $2, $3, 0, $4, $5, $6)",
+ )
+ .bind(id)
+ .bind(kb)
+ .bind(document)
+ .bind(text)
+ .bind(vector)
+ .bind(t("2026-02-01T00:00:00Z"))
+ .execute(pool)
+ .await?;
+ }
+
+ // 五月:重解析。旧块落选被软删,新块顶上——**旧块的向量必须还在**
+ let pieces = vec![ChunkPiece {
+ seq: 0,
+ text: "The 2026 handbook says the office moved to Shenzhen.".into(),
+ char_start: 0,
+ char_end: 51,
+ }];
+ utopia_store::documents::replace_chunks(pool, kb, doc, &pieces).await?;
+ let new_chunk: (Uuid,) = sqlx::query_as(
+ "SELECT id FROM chunks WHERE document_id = $1 AND superseded_at IS NULL LIMIT 1",
+ )
+ .bind(doc)
+ .fetch_one(pool)
+ .await?;
+ // 新块的向量由摄入管道后补,这里直接给一个,好让它进得了向量召回
+ sqlx::query("UPDATE chunks SET embedding = $2, created_at = $3 WHERE id = $1")
+ .bind(new_chunk.0)
+ .bind(embed([0.0, 1.0, 0.0]))
+ .bind(t("2026-05-01T00:00:00Z"))
+ .execute(pool)
+ .await?;
+ sqlx::query("UPDATE chunks SET superseded_at = $2 WHERE id = $1")
+ .bind(old_chunk)
+ .bind(t("2026-05-01T00:00:00Z"))
+ .execute(pool)
+ .await?;
+
+ Ok(Fixture {
+ org,
+ kb,
+ old_chunk,
+ new_chunk: new_chunk.0,
+ deleted_doc_chunk,
+ })
+}
+
+#[tokio::test]
+async fn a_superseded_chunk_keeps_its_vector_and_answers_as_of_then() -> anyhow::Result<()> {
+ let Some(url) = utopia_store::test_db::url() else {
+ return Ok(());
+ };
+ let pool = PgPool::connect(&url).await?;
+ let f = seed(&pool).await?;
+
+ // 1. 顶掉的块还带着向量。丢了向量,下面每一条都无从谈起
+ let kept: (bool,) = sqlx::query_as("SELECT embedding IS NOT NULL FROM chunks WHERE id = $1")
+ .bind(f.old_chunk)
+ .fetch_one(&pool)
+ .await?;
+ assert!(
+ kept.0,
+ "重解析不该顺手丢掉旧块的向量——历史就是靠它才搜得出来"
+ );
+
+ // 2. 现在:向量召回给出新块,给不出被顶掉的旧块
+ let query = vec![1.0_f32, 0.0, 0.0];
+ let now_hits = utopia_store::documents::vector_search(&pool, f.kb, &query, 10, None).await?;
+ assert!(!now_hits.contains(&f.old_chunk), "现行检索不该翻出旧版本");
+ assert!(now_hits.contains(&f.new_chunk));
+
+ // 3. 三月:那时旧块是现行的,新块还不存在
+ let then = Some(t("2026-03-01T00:00:00Z"));
+ let then_hits = utopia_store::documents::vector_search(&pool, f.kb, &query, 10, then).await?;
+ assert!(
+ then_hits.contains(&f.old_chunk),
+ "三月该搜得到当时那一版——这一条在丢向量的实现上不可能成立"
+ );
+ assert!(
+ !then_hits.contains(&f.new_chunk),
+ "五月才有的块不该出现在三月"
+ );
+
+ // 4. 取块也倒:六月删掉的文档,在五月的检索里照常带回来(#268 留了墓碑)
+ let live_now = utopia_store::documents::chunks_by_ids(
+ &pool,
+ f.kb,
+ &[f.deleted_doc_chunk, f.new_chunk],
+ None,
+ )
+ .await?;
+ let ids: Vec<Uuid> = live_now.iter().map(|c| c.id).collect();
+ assert!(
+ !ids.contains(&f.deleted_doc_chunk),
+ "删掉的文档不出现在现在"
+ );
+ assert!(ids.contains(&f.new_chunk));
+
+ let live_then = utopia_store::documents::chunks_by_ids(
+ &pool,
+ f.kb,
+ &[f.deleted_doc_chunk, f.old_chunk],
+ Some(t("2026-05-15T00:00:00Z")),
+ )
+ .await?;
+ let ids: Vec<Uuid> = live_then.iter().map(|c| c.id).collect();
+ assert!(
+ ids.contains(&f.deleted_doc_chunk),
+ "五月中那篇文档还没被删,它的块该拿得回来"
+ );
+ assert!(
+ !ids.contains(&f.old_chunk),
+ "五月初已被顶掉的块不属于五月中"
+ );
+
+ sqlx::query("DELETE FROM knowledge_bases WHERE id = $1")
+ .bind(f.kb)
+ .execute(&pool)
+ .await?;
+ let gone = sqlx::query("DELETE FROM organizations WHERE id = $1")
+ .bind(f.org)
+ .execute(&pool)
+ .await?;
+ assert_eq!(gone.rows_affected(), 1, "一次性 org 没删掉");
+ Ok(())
+}
diff --git a/docs/decisions/0019-the-second-clock-can-be-rewound.md b/docs/decisions/0019-the-second-clock-can-be-rewound.md
index 95dfcec8..f6871307 100644
--- a/docs/decisions/0019-the-second-clock-can-be-rewound.md
+++ b/docs/decisions/0019-the-second-clock-can-be-rewound.md
@@ -1,6 +1,6 @@
# 0019 · The second clock can be rewound
-- **Status**: planned · nothing built; the rows are already there, the read paths are not
+- **Status**: implemented in three cuts · #317 gave every graph read the `held_at` predicate and `as_of` beside `at`; #337 gave entities their clock by unwinding `entity_merges`; this record's second open question landed with retrieval (superseded chunks keep their vectors, and vector recall and chunk fetch both take `as_of`) · the control on the graph page is still open, and full-text recall is still "now" only
- **Written**: 2026-09-04 (conventions in the [README](README.md))
- **Related**: [0003](0003-ontology-growth-loop.md) put adoption's rewrites on the same append path as human correction, so the prior state is still on disk; [0002](0002-reasoning-engine.md) built the proof chain on the same rows. #268 (deleting a document) is what made the gap urgent, and is deliberately a separate change
@@ -34,6 +34,6 @@ Read-only throughout. No new column, no migration.
## Open questions
-- **Entities have no clock.** `merged_into` records that a merge happened, not when; the time is in `entity_merges.created_at` / `reverted_at`. Two entities merged in March should be two nodes at an `as_of` in February, which means unwinding merges through that table rather than reading the entity row. Probably a second cut.
-- **Retrieval has the same two clocks.** `chunks` carries `created_at` / `superseded_at`, so the predicate above applies unchanged. What is missing is content: `replace_chunks` clears `embedding` on the superseded version to save storage, and the full-text index keeps no history. Deleted documents keep everything (#268), so as-of retrieval over them costs nothing; over earlier *versions* it costs keeping their vectors, which is a storage decision, and full text stays "now" either way.
+- **Entities have no clock.** ~~`merged_into` records that a merge happened, not when.~~ Settled in #337: `entity_merges.created_at` / `reverted_at` is the clock, `fact_owner_at` reads the moved-fact arrays back, and a merge made in March leaves two nodes at an `as_of` in February. The target's own type and profile at T are recorded (`target_type_before`, `target_profile_before`) and still unread.
+- **Retrieval has the same two clocks.** Settled, with one half left standing. `replace_chunks` no longer clears `embedding` on a superseded chunk: the storage that decision saved was the smaller half — the chunk's **text** was already kept — and the price was that history could not be searched at all. Vector recall and chunk fetch now take `as_of`, and a document deleted after T comes back with its chunks (#268 leaves the tombstone). **Full text stays "now"**: Tantivy holds one version of a base, so a timed search returns correct hits and misses the ones only history has. Giving the index versions is a separate piece of work, not a filter.
- **What the control is.** A second slider doubles the surface for a question most people ask rarely; a mode switch on the existing slider is cheaper but risks reading as the same axis, which is the confusion the section above is written to prevent.