World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
A page never skips a row (#656)
WaylandYang2d743bd40dc6
diff --git a/crates/utopia-store/src/alerts.rs b/crates/utopia-store/src/alerts.rsindex dca115ee..dfa110eb 100644--- a/crates/utopia-store/src/alerts.rs+++ b/crates/utopia-store/src/alerts.rs@@ -219,7 +219,7 @@ pub async fn list_groups((array_agg(detail ORDER BY created_at DESC))[1:{GROUP_LINES}] AS linesFROM islGROUP BY kb_id, kind, grp- ORDER BY max(created_at) DESC+ ORDER BY max(created_at) DESC, kb_id, kind, grpLIMIT $6 OFFSET $7");let items: Vec<AlertGroup> = sqlx::query_as(&sql)diff --git a/crates/utopia-store/src/audit.rs b/crates/utopia-store/src/audit.rsindex 9cd1e63e..64ae8c89 100644--- a/crates/utopia-store/src/audit.rs+++ b/crates/utopia-store/src/audit.rs@@ -103,7 +103,7 @@ pub async fn review_history("SELECT e.id, e.action, e.target_kind, e.target_id, e.detail,e.actor_id, u.display_name AS actor_name, e.created_atFROM audit_events e LEFT JOIN users u ON u.id = e.actor_id- WHERE {COND} ORDER BY e.created_at DESC LIMIT $2 OFFSET $3"+ WHERE {COND} ORDER BY e.created_at DESC, e.id DESC LIMIT $2 OFFSET $3")).bind(kb_id).bind(limit)@@ -155,7 +155,7 @@ pub async fn list_for_kb(e.actor_id, u.display_name AS actor_name, e.created_atFROM audit_events e LEFT JOIN users u ON u.id = e.actor_id{WHERE}- ORDER BY e.created_at DESC LIMIT $6 OFFSET $7"+ ORDER BY e.created_at DESC, e.id DESC LIMIT $6 OFFSET $7")).bind(kb_id).bind(action)diff --git a/crates/utopia-store/src/business_rules.rs b/crates/utopia-store/src/business_rules.rsindex 13694786..fae8b26d 100644--- a/crates/utopia-store/src/business_rules.rs+++ b/crates/utopia-store/src/business_rules.rs@@ -421,7 +421,7 @@ pub async fn matches(JOIN attribute_rules ar ON ar.id = d.attribute_rule_idLEFT JOIN entity_types ct ON ct.id = ar.conclude_type_idWHERE d.kb_id = $1 AND d.attribute_rule_id = $2 AND d.invalidated_at IS NULL- ORDER BY e.canonical_name, d.valid_from+ ORDER BY e.canonical_name, d.valid_from, d.idLIMIT $3 OFFSET $4",).bind(kb_id)diff --git a/crates/utopia-store/src/conversations.rs b/crates/utopia-store/src/conversations.rsindex e76d8156..541ded63 100644--- a/crates/utopia-store/src/conversations.rs+++ b/crates/utopia-store/src/conversations.rs@@ -44,7 +44,7 @@ pub async fn list(WHERE m.conversation_id = c.id) AS message_countFROM conversations c{WHERE}- ORDER BY c.updated_at DESC+ ORDER BY c.updated_at DESC, c.id DESCLIMIT $4 OFFSET $5")).bind(kb_id)diff --git a/crates/utopia-store/src/documents.rs b/crates/utopia-store/src/documents.rsindex e34c8a3c..d48597be 100644--- a/crates/utopia-store/src/documents.rs+++ b/crates/utopia-store/src/documents.rs@@ -399,7 +399,7 @@ pub async fn page(let docs: Vec<Document> = sqlx::query_as(&format!("SELECT * FROM documents {WHERE}- ORDER BY (CASE WHEN $5::bool THEN deleted_at ELSE created_at END) DESC+ ORDER BY (CASE WHEN $5::bool THEN deleted_at ELSE created_at END) DESC, id DESCLIMIT $6 OFFSET $7")).bind(kb_id)diff --git a/crates/utopia-store/src/graph.rs b/crates/utopia-store/src/graph.rsindex 83bf9369..0c075bcd 100644--- a/crates/utopia-store/src/graph.rs+++ b/crates/utopia-store/src/graph.rs@@ -1082,7 +1082,7 @@ pub async fn search_entities("{} WHERE e.kb_id = $1 AND e.merged_into IS NULLAND (e.canonical_name ILIKE $2OR EXISTS (SELECT 1 FROM unnest(e.aliases) AS a WHERE a ILIKE $2))- ORDER BY degree DESC, e.canonical_name LIMIT $3 OFFSET $4",+ ORDER BY degree DESC, e.canonical_name, e.id LIMIT $3 OFFSET $4",node_sql(None, None))).bind(kb_id)@@ -1343,7 +1343,7 @@ pub async fn low_confidence_facts(LEFT JOIN relation_types r ON r.id = f.predicate_idLEFT JOIN entities o ON o.id = f.object_idWHERE f.kb_id = $1 AND f.invalidated_at IS NULL AND f.confidence < $2- ORDER BY f.confidence, f.recorded_at DESC+ ORDER BY f.confidence, f.recorded_at DESC, f.id DESCLIMIT $3 OFFSET $4",).bind(kb_id)@@ -1380,7 +1380,7 @@ pub async fn stale_facts(AND NOT EXISTS (SELECT 1 FROM fact_evidence feJOIN chunks c ON c.id = fe.chunk_idWHERE fe.fact_id = f.id AND c.superseded_at IS NULL)- ORDER BY f.recorded_at DESC+ ORDER BY f.recorded_at DESC, f.id DESCLIMIT $2 OFFSET $3",).bind(kb_id)@@ -1679,7 +1679,9 @@ pub async fn entity_history(LEFT JOIN entities o ON o.id = mg.other_idLEFT JOIN users u ON u.id = mg.actor_id) x- ORDER BY x.at DESC, x.fact_id+ -- 改类型与合并那两支没有 fact_id,同一刻的几行只靠它排不出先后(#646):+ -- 再按种类,最后按整行——两行连整行都一样,谁先谁后看不出差别+ ORDER BY x.at DESC, x.fact_id, x.kind, x::textLIMIT $3 OFFSET $4")).bind(kb_id)diff --git a/crates/utopia-store/src/mappings.rs b/crates/utopia-store/src/mappings.rsindex c7d772eb..1f5f5e20 100644--- a/crates/utopia-store/src/mappings.rs+++ b/crates/utopia-store/src/mappings.rs@@ -144,7 +144,7 @@ pub async fn proposed(FROM concept_mappings mJOIN entities e ON e.id = m.concept_idWHERE m.kb_id = $1 AND m.status = 'proposed'- ORDER BY e.canonical_name, m.source+ ORDER BY e.canonical_name, m.source, m.idLIMIT $2 OFFSET $3",).bind(kb_id)@@ -288,7 +288,7 @@ pub async fn page(FROM concept_mappings mJOIN entities e ON e.id = m.concept_id{WHERE}- ORDER BY e.canonical_name, m.source+ ORDER BY e.canonical_name, m.source, m.idLIMIT $4 OFFSET $5")).bind(kb_id)diff --git a/crates/utopia-store/src/ontology.rs b/crates/utopia-store/src/ontology.rsindex 24eac774..79a8c210 100644--- a/crates/utopia-store/src/ontology.rs+++ b/crates/utopia-store/src/ontology.rs@@ -42,7 +42,7 @@ pub async fn entity_instances(AND f.invalidated_at IS NULL) AS fact_countFROM entities eWHERE e.kb_id = $1 AND e.type_id = $2 AND e.merged_into IS NULL- ORDER BY lower(e.canonical_name)+ ORDER BY lower(e.canonical_name), e.idLIMIT $3 OFFSET $4",).bind(kb_id)diff --git a/crates/utopia-store/src/pending.rs b/crates/utopia-store/src/pending.rsindex 4a406522..36510292 100644--- a/crates/utopia-store/src/pending.rs+++ b/crates/utopia-store/src/pending.rs@@ -160,7 +160,7 @@ pub async fn list(offset: i64,) -> AppResult<Vec<PendingFactView>> {Ok(sqlx::query_as(&format!(- "{VIEW_SELECT} WHERE p.kb_id = $1 ORDER BY p.created_at DESC LIMIT $2 OFFSET $3"+ "{VIEW_SELECT} WHERE p.kb_id = $1 ORDER BY p.created_at DESC, p.id DESC LIMIT $2 OFFSET $3")).bind(kb_id).bind(limit)diff --git a/crates/utopia-store/src/reasoning.rs b/crates/utopia-store/src/reasoning.rsindex 26b40473..4e7334ff 100644--- a/crates/utopia-store/src/reasoning.rs+++ b/crates/utopia-store/src/reasoning.rs@@ -617,7 +617,9 @@ pub async fn open_violations(JOIN triple rt ON rt.id = v.right_factJOIN facts lf ON lf.id = v.left_factWHERE v.kb_id = $1 AND v.status = 'open'- ORDER BY v.detected_at DESC+ -- id 做第二键(#646):一轮检查插下的行 detected_at 全都相同,只按它排,+ -- 翻页时每一页的先后可以不同——有的行出现两次,有的一次也不出现+ ORDER BY v.detected_at DESC, v.id DESCLIMIT $2 OFFSET $3",// 「左边还开着」按读出来的终点判(0022):结束了不知哪天的不算开着left_holds_to = crate::world_axis::facts_holds_to("lf"),@@ -2116,7 +2118,7 @@ pub async fn open_defects(LEFT JOIN entity_types ot ON ot.id = d.otherLEFT JOIN relation_types orr ON orr.id = d.otherWHERE d.kb_id = $1 AND d.status = 'open'- ORDER BY d.detected_at DESC+ ORDER BY d.detected_at DESC, d.id DESCLIMIT $2 OFFSET $3",).bind(kb_id)diff --git a/crates/utopia-store/src/resolution.rs b/crates/utopia-store/src/resolution.rsindex 22650da2..eee15853 100644--- a/crates/utopia-store/src/resolution.rs+++ b/crates/utopia-store/src/resolution.rs@@ -1415,7 +1415,7 @@ pub async fn list_reviews(JOIN entities a ON a.id = rr.left_idJOIN entities b ON b.id = rr.right_idWHERE rr.kb_id = $1 AND rr.status = 'pending' AND {}- ORDER BY rr.created_at DESC LIMIT $2 OFFSET $3",+ ORDER BY rr.created_at DESC, rr.id DESC LIMIT $2 OFFSET $3",types.clause());let rows: Vec<ReviewRow> = sqlx::query_as(&sql)@@ -2040,7 +2040,7 @@ pub async fn list_merges(JOIN entities t ON t.id = m.target_idLEFT JOIN users u ON u.id = m.merged_byWHERE m.kb_id = $1- ORDER BY m.created_at DESC LIMIT $2 OFFSET $3",+ ORDER BY m.created_at DESC, m.id DESC LIMIT $2 OFFSET $3",).bind(kb_id).bind(limit)diff --git a/crates/utopia-store/src/temporal.rs b/crates/utopia-store/src/temporal.rsindex 236d60fe..05ee9087 100644--- a/crates/utopia-store/src/temporal.rs+++ b/crates/utopia-store/src/temporal.rs@@ -552,7 +552,7 @@ pub async fn list_conflicts(LEFT JOIN entities oo ON oo.id = fo.object_idLEFT JOIN entities no_ ON no_.id = fn_.object_idWHERE c.kb_id = $1 AND c.status = 'open'- ORDER BY c.created_at DESC+ ORDER BY c.created_at DESC, c.id DESCLIMIT $2 OFFSET $3",).bind(kb_id)diff --git a/crates/utopia-store/tests/a_page_never_skips_a_row.rs b/crates/utopia-store/tests/a_page_never_skips_a_row.rsnew file mode 100644index 00000000..03977478--- /dev/null+++ b/crates/utopia-store/tests/a_page_never_skips_a_row.rs@@ -0,0 +1,224 @@+//! 分页翻完,每一行恰好出现一次(#646),打在真库上。+//!+//! Review 的各档与台账都是 `LIMIT/OFFSET` 分页。排序键若不唯一,并列的行每次查询的+//! 先后可以不同——翻到第二页时第一页见过的又出来一次,另一行一页也没出现。时间戳+//! 作排序键时,逐条写入的行很少并列;**一个事务里写下的行全都并列**(`now()` 在事务+//! 里是同一个值),而 Review 的队列正是成批写的。量过:一次检查插下 572 个环,按 200 条+//! 一页翻,翻出 557 个。+//!+//! 三份成批写下的数据,各翻一遍:+//!+//! 1. **公理违规**:一次 `reasoning::run` 插下几百个环,`open_violations` 翻+//! 2. **本体缺陷**:一个语句插下几百行,`open_defects` 翻+//! 3. **审核台账**:一个语句写下几百条审计事件,`review_history` 翻+//!+//! 断言的不只是「每行一次」,而是**翻出来的序列恰好是按时间倒序、再按 id 倒序排好的+//! 全部行**。只断言每行一次是靠运气的:并列行的先后取决于执行计划,旧的排序在这份+//! 数据上三次里只错一次。行的 id 是随机的,与写入的物理顺序无关,所以只要排序在并列+//! 处没有定下来,序列就对不上——每次都对不上。+//!+//! 页长取质数(37),页边界落不到整齐的地方。没有 `UTOPIA_DATABASE_URL` 时跳过而不是+//! 失败。自建自拆,绝不碰已有的库。++use sqlx::PgPool;+use std::future::Future;+use utopia_store::reasoning;+use uuid::Uuid;++const PAGE: i64 = 37;++struct Fixture {+ org: Uuid,+ kb: Uuid,+}++async fn seed(pool: &PgPool) -> anyhow::Result<Fixture> {+ let (org, ws, kb) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7());+ sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, 'paging-test')")+ .bind(org)+ .execute(pool)+ .await?;+ sqlx::query("INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, 'paging-test')")+ .bind(ws)+ .bind(org)+ .execute(pool)+ .await?;+ sqlx::query(+ "INSERT INTO knowledge_bases (id, workspace_id, name) VALUES ($1, $2, 'paging-test')",+ )+ .bind(kb)+ .bind(ws)+ .execute(pool)+ .await?;+ Ok(Fixture { org, kb })+}++/// 一页一页翻到底,按翻到的先后接起来+async fn page_through<F, Fut>(mut page: F) -> anyhow::Result<Vec<Uuid>>+where+ F: FnMut(i64) -> Fut,+ Fut: Future<Output = anyhow::Result<Vec<Uuid>>>,+{+ let mut seen = Vec::new();+ let mut offset = 0;+ loop {+ let ids = page(offset).await?;+ let n = ids.len() as i64;+ seen.extend(ids);+ if n < PAGE {+ return Ok(seen);+ }+ offset += PAGE;+ }+}++/// 翻出来的就是全部行,一行一次,并且时间戳并列时按 id 倒序——排序是全序+async fn assert_total_order(+ pool: &PgPool,+ what: &str,+ seen: &[Uuid],+ all_ids: &str,+ expected: usize,+) -> anyhow::Result<()> {+ let mut want: Vec<Uuid> = sqlx::query_scalar(all_ids).fetch_all(pool).await?;+ assert_eq!(want.len(), expected, "{what}: 写下的行数");+ want.sort_by(|a, b| b.cmp(a));+ let mut distinct = seen.to_vec();+ distinct.sort();+ distinct.dedup();+ assert_eq!(+ distinct.len(),+ seen.len(),+ "{what}: {} 行翻到了不止一次",+ seen.len() - distinct.len()+ );+ assert_eq!(seen.len(), expected, "{what}: 翻完只见到 {} 行", seen.len());+ assert!(+ seen == want,+ "{what}: 并列的行没有按 id 定下先后,翻页之间会漂"+ );+ Ok(())+}++#[tokio::test]+async fn paging_through_a_batch_sees_every_row_exactly_once() -> anyhow::Result<()> {+ let Some(url) = utopia_store::test_db::url() else {+ return Ok(());+ };+ let pool = PgPool::connect(&url).await?;+ let f = seed(&pool).await?;++ let result = async {+ // ---- 一、公理违规:一个中心节点与 300 个节点两两互指,恒常谓词上就是 300 个环,+ // 一次检查、一个事务插下+ const CYCLES: usize = 300;+ let (etype, part_of) = (Uuid::now_v7(), Uuid::now_v7());+ sqlx::query("INSERT INTO entity_types (id, kb_id, key, label) VALUES ($1, $2, 't', 'T')")+ .bind(etype)+ .bind(f.kb)+ .execute(&pool)+ .await?;+ sqlx::query(+ "INSERT INTO relation_types (id, kb_id, key, label, temporal, is_transitive)+ VALUES ($1, $2, 'part_of', 'part_of', 'eternal', TRUE)",+ )+ .bind(part_of)+ .bind(f.kb)+ .execute(&pool)+ .await?;+ sqlx::query(+ "WITH n AS (+ INSERT INTO entities (id, kb_id, type_id, canonical_name)+ SELECT gen_random_uuid(), $1, $2, 'N' || i FROM generate_series(0, $4) AS i+ RETURNING id, canonical_name+ ), hub AS (SELECT id FROM n WHERE canonical_name = 'N0'),+ spokes AS (SELECT id FROM n WHERE canonical_name <> 'N0')+ INSERT INTO facts (id, kb_id, subject_id, predicate_id, object_id)+ SELECT gen_random_uuid(), $1, hub.id, $3, spokes.id FROM hub, spokes+ UNION ALL+ SELECT gen_random_uuid(), $1, spokes.id, $3, hub.id FROM hub, spokes",+ )+ .bind(f.kb)+ .bind(etype)+ .bind(part_of)+ .bind(CYCLES as i32)+ .execute(&pool)+ .await?;+ let r = reasoning::run(&pool, f.kb).await?;+ assert_eq!(r.inserted, CYCLES, "一次检查插下这些环");+ let seen = page_through(|offset| {+ let pool = pool.clone();+ async move {+ Ok(reasoning::open_violations(&pool, f.kb, PAGE, offset)+ .await?+ .into_iter()+ .map(|v| v.id)+ .collect())+ }+ })+ .await?;+ let ids = format!("SELECT id FROM axiom_violations WHERE kb_id = '{}'", f.kb);+ assert_total_order(&pool, "公理违规", &seen, &ids, CYCLES).await?;++ // ---- 二、本体缺陷:一个语句插下 250 行,detected_at 全相同+ const DEFECTS: usize = 250;+ sqlx::query(+ "INSERT INTO ontology_defects (id, kb_id, kind, subject, other)+ SELECT gen_random_uuid(), $1, 'rules_disagree', gen_random_uuid(), gen_random_uuid()+ FROM generate_series(1, $2)",+ )+ .bind(f.kb)+ .bind(DEFECTS as i32)+ .execute(&pool)+ .await?;+ let seen = page_through(|offset| {+ let pool = pool.clone();+ async move {+ Ok(reasoning::open_defects(&pool, f.kb, PAGE, offset)+ .await?+ .into_iter()+ .map(|d| d.id)+ .collect())+ }+ })+ .await?;+ let ids = format!("SELECT id FROM ontology_defects WHERE kb_id = '{}'", f.kb);+ assert_total_order(&pool, "本体缺陷", &seen, &ids, DEFECTS).await?;++ // ---- 三、审核台账:一个语句写下 250 条,created_at 全相同+ const EVENTS: usize = 250;+ sqlx::query(+ "INSERT INTO audit_events (id, kb_id, action, target_kind)+ SELECT gen_random_uuid(), $1, 'review.decided', 'violation'+ FROM generate_series(1, $2)",+ )+ .bind(f.kb)+ .bind(EVENTS as i32)+ .execute(&pool)+ .await?;+ let seen = page_through(|offset| {+ let pool = pool.clone();+ async move {+ Ok(+ utopia_store::audit::review_history(&pool, f.kb, PAGE, offset)+ .await?+ .0+ .into_iter()+ .map(|e| e.id)+ .collect(),+ )+ }+ })+ .await?;+ let ids = format!("SELECT id FROM audit_events WHERE kb_id = '{}'", f.kb);+ assert_total_order(&pool, "审核台账", &seen, &ids, EVENTS).await?;+ Ok::<_, anyhow::Error>(())+ }+ .await;+ // 审计事件只许追加(删不掉),留在测试库里;它们挂在这个自建的库名下,不串到别处+ sqlx::query("DELETE FROM organizations WHERE id = $1")+ .bind(f.org)+ .execute(&pool)+ .await?;+ result+}