Skip to content

World's first open-source enterprise world model.

A page never skips a row (#656)

WaylandYang2d743bd40dc6
diff --git a/crates/utopia-store/src/alerts.rs b/crates/utopia-store/src/alerts.rs
index dca115ee..dfa110eb 100644
--- a/crates/utopia-store/src/alerts.rs
+++ b/crates/utopia-store/src/alerts.rs
@@ -219,7 +219,7 @@ pub async fn list_groups(
(array_agg(detail ORDER BY created_at DESC))[1:{GROUP_LINES}] AS lines
FROM isl
GROUP BY kb_id, kind, grp
- ORDER BY max(created_at) DESC
+ ORDER BY max(created_at) DESC, kb_id, kind, grp
LIMIT $6 OFFSET $7"
);
let items: Vec<AlertGroup> = sqlx::query_as(&sql)
diff --git a/crates/utopia-store/src/audit.rs b/crates/utopia-store/src/audit.rs
index 9cd1e63e..64ae8c89 100644
--- a/crates/utopia-store/src/audit.rs
+++ b/crates/utopia-store/src/audit.rs
@@ -103,7 +103,7 @@ pub async fn review_history(
"SELECT e.id, e.action, e.target_kind, e.target_id, e.detail,
e.actor_id, u.display_name AS actor_name, e.created_at
FROM audit_events e LEFT JOIN users u ON u.id = e.actor_id
- WHERE {COND} ORDER BY e.created_at DESC LIMIT $2 OFFSET $3"
+ WHERE {COND} ORDER BY e.created_at DESC, e.id DESC LIMIT $2 OFFSET $3"
))
.bind(kb_id)
.bind(limit)
@@ -155,7 +155,7 @@ pub async fn list_for_kb(
e.actor_id, u.display_name AS actor_name, e.created_at
FROM audit_events e LEFT JOIN users u ON u.id = e.actor_id
{WHERE}
- ORDER BY e.created_at DESC LIMIT $6 OFFSET $7"
+ ORDER BY e.created_at DESC, e.id DESC LIMIT $6 OFFSET $7"
))
.bind(kb_id)
.bind(action)
diff --git a/crates/utopia-store/src/business_rules.rs b/crates/utopia-store/src/business_rules.rs
index 13694786..fae8b26d 100644
--- a/crates/utopia-store/src/business_rules.rs
+++ b/crates/utopia-store/src/business_rules.rs
@@ -421,7 +421,7 @@ pub async fn matches(
JOIN attribute_rules ar ON ar.id = d.attribute_rule_id
LEFT JOIN entity_types ct ON ct.id = ar.conclude_type_id
WHERE d.kb_id = $1 AND d.attribute_rule_id = $2 AND d.invalidated_at IS NULL
- ORDER BY e.canonical_name, d.valid_from
+ ORDER BY e.canonical_name, d.valid_from, d.id
LIMIT $3 OFFSET $4",
)
.bind(kb_id)
diff --git a/crates/utopia-store/src/conversations.rs b/crates/utopia-store/src/conversations.rs
index e76d8156..541ded63 100644
--- a/crates/utopia-store/src/conversations.rs
+++ b/crates/utopia-store/src/conversations.rs
@@ -44,7 +44,7 @@ pub async fn list(
WHERE m.conversation_id = c.id) AS message_count
FROM conversations c
{WHERE}
- ORDER BY c.updated_at DESC
+ ORDER BY c.updated_at DESC, c.id DESC
LIMIT $4 OFFSET $5"
))
.bind(kb_id)
diff --git a/crates/utopia-store/src/documents.rs b/crates/utopia-store/src/documents.rs
index e34c8a3c..d48597be 100644
--- a/crates/utopia-store/src/documents.rs
+++ b/crates/utopia-store/src/documents.rs
@@ -399,7 +399,7 @@ pub async fn page(
let docs: Vec<Document> = sqlx::query_as(&format!(
"SELECT * FROM documents {WHERE}
- ORDER BY (CASE WHEN $5::bool THEN deleted_at ELSE created_at END) DESC
+ ORDER BY (CASE WHEN $5::bool THEN deleted_at ELSE created_at END) DESC, id DESC
LIMIT $6 OFFSET $7"
))
.bind(kb_id)
diff --git a/crates/utopia-store/src/graph.rs b/crates/utopia-store/src/graph.rs
index 83bf9369..0c075bcd 100644
--- a/crates/utopia-store/src/graph.rs
+++ b/crates/utopia-store/src/graph.rs
@@ -1082,7 +1082,7 @@ pub async fn search_entities(
"{} WHERE e.kb_id = $1 AND e.merged_into IS NULL
AND (e.canonical_name ILIKE $2
OR EXISTS (SELECT 1 FROM unnest(e.aliases) AS a WHERE a ILIKE $2))
- ORDER BY degree DESC, e.canonical_name LIMIT $3 OFFSET $4",
+ ORDER BY degree DESC, e.canonical_name, e.id LIMIT $3 OFFSET $4",
node_sql(None, None)
))
.bind(kb_id)
@@ -1343,7 +1343,7 @@ pub async fn low_confidence_facts(
LEFT JOIN relation_types r ON r.id = f.predicate_id
LEFT JOIN entities o ON o.id = f.object_id
WHERE f.kb_id = $1 AND f.invalidated_at IS NULL AND f.confidence < $2
- ORDER BY f.confidence, f.recorded_at DESC
+ ORDER BY f.confidence, f.recorded_at DESC, f.id DESC
LIMIT $3 OFFSET $4",
)
.bind(kb_id)
@@ -1380,7 +1380,7 @@ pub async fn stale_facts(
AND NOT EXISTS (SELECT 1 FROM fact_evidence fe
JOIN chunks c ON c.id = fe.chunk_id
WHERE fe.fact_id = f.id AND c.superseded_at IS NULL)
- ORDER BY f.recorded_at DESC
+ ORDER BY f.recorded_at DESC, f.id DESC
LIMIT $2 OFFSET $3",
)
.bind(kb_id)
@@ -1679,7 +1679,9 @@ pub async fn entity_history(
LEFT JOIN entities o ON o.id = mg.other_id
LEFT JOIN users u ON u.id = mg.actor_id
) x
- ORDER BY x.at DESC, x.fact_id
+ -- 改类型与合并那两支没有 fact_id,同一刻的几行只靠它排不出先后(#646):
+ -- 再按种类,最后按整行——两行连整行都一样,谁先谁后看不出差别
+ ORDER BY x.at DESC, x.fact_id, x.kind, x::text
LIMIT $3 OFFSET $4"
))
.bind(kb_id)
diff --git a/crates/utopia-store/src/mappings.rs b/crates/utopia-store/src/mappings.rs
index c7d772eb..1f5f5e20 100644
--- a/crates/utopia-store/src/mappings.rs
+++ b/crates/utopia-store/src/mappings.rs
@@ -144,7 +144,7 @@ pub async fn proposed(
FROM concept_mappings m
JOIN entities e ON e.id = m.concept_id
WHERE m.kb_id = $1 AND m.status = 'proposed'
- ORDER BY e.canonical_name, m.source
+ ORDER BY e.canonical_name, m.source, m.id
LIMIT $2 OFFSET $3",
)
.bind(kb_id)
@@ -288,7 +288,7 @@ pub async fn page(
FROM concept_mappings m
JOIN entities e ON e.id = m.concept_id
{WHERE}
- ORDER BY e.canonical_name, m.source
+ ORDER BY e.canonical_name, m.source, m.id
LIMIT $4 OFFSET $5"
))
.bind(kb_id)
diff --git a/crates/utopia-store/src/ontology.rs b/crates/utopia-store/src/ontology.rs
index 24eac774..79a8c210 100644
--- a/crates/utopia-store/src/ontology.rs
+++ b/crates/utopia-store/src/ontology.rs
@@ -42,7 +42,7 @@ pub async fn entity_instances(
AND f.invalidated_at IS NULL) AS fact_count
FROM entities e
WHERE e.kb_id = $1 AND e.type_id = $2 AND e.merged_into IS NULL
- ORDER BY lower(e.canonical_name)
+ ORDER BY lower(e.canonical_name), e.id
LIMIT $3 OFFSET $4",
)
.bind(kb_id)
diff --git a/crates/utopia-store/src/pending.rs b/crates/utopia-store/src/pending.rs
index 4a406522..36510292 100644
--- a/crates/utopia-store/src/pending.rs
+++ b/crates/utopia-store/src/pending.rs
@@ -160,7 +160,7 @@ pub async fn list(
offset: i64,
) -> AppResult<Vec<PendingFactView>> {
Ok(sqlx::query_as(&format!(
- "{VIEW_SELECT} WHERE p.kb_id = $1 ORDER BY p.created_at DESC LIMIT $2 OFFSET $3"
+ "{VIEW_SELECT} WHERE p.kb_id = $1 ORDER BY p.created_at DESC, p.id DESC LIMIT $2 OFFSET $3"
))
.bind(kb_id)
.bind(limit)
diff --git a/crates/utopia-store/src/reasoning.rs b/crates/utopia-store/src/reasoning.rs
index 26b40473..4e7334ff 100644
--- a/crates/utopia-store/src/reasoning.rs
+++ b/crates/utopia-store/src/reasoning.rs
@@ -617,7 +617,9 @@ pub async fn open_violations(
JOIN triple rt ON rt.id = v.right_fact
JOIN facts lf ON lf.id = v.left_fact
WHERE v.kb_id = $1 AND v.status = 'open'
- ORDER BY v.detected_at DESC
+ -- id 做第二键(#646):一轮检查插下的行 detected_at 全都相同,只按它排,
+ -- 翻页时每一页的先后可以不同——有的行出现两次,有的一次也不出现
+ ORDER BY v.detected_at DESC, v.id DESC
LIMIT $2 OFFSET $3",
// 「左边还开着」按读出来的终点判(0022):结束了不知哪天的不算开着
left_holds_to = crate::world_axis::facts_holds_to("lf"),
@@ -2116,7 +2118,7 @@ pub async fn open_defects(
LEFT JOIN entity_types ot ON ot.id = d.other
LEFT JOIN relation_types orr ON orr.id = d.other
WHERE d.kb_id = $1 AND d.status = 'open'
- ORDER BY d.detected_at DESC
+ ORDER BY d.detected_at DESC, d.id DESC
LIMIT $2 OFFSET $3",
)
.bind(kb_id)
diff --git a/crates/utopia-store/src/resolution.rs b/crates/utopia-store/src/resolution.rs
index 22650da2..eee15853 100644
--- a/crates/utopia-store/src/resolution.rs
+++ b/crates/utopia-store/src/resolution.rs
@@ -1415,7 +1415,7 @@ pub async fn list_reviews(
JOIN entities a ON a.id = rr.left_id
JOIN entities b ON b.id = rr.right_id
WHERE rr.kb_id = $1 AND rr.status = 'pending' AND {}
- ORDER BY rr.created_at DESC LIMIT $2 OFFSET $3",
+ ORDER BY rr.created_at DESC, rr.id DESC LIMIT $2 OFFSET $3",
types.clause()
);
let rows: Vec<ReviewRow> = sqlx::query_as(&sql)
@@ -2040,7 +2040,7 @@ pub async fn list_merges(
JOIN entities t ON t.id = m.target_id
LEFT JOIN users u ON u.id = m.merged_by
WHERE m.kb_id = $1
- ORDER BY m.created_at DESC LIMIT $2 OFFSET $3",
+ ORDER BY m.created_at DESC, m.id DESC LIMIT $2 OFFSET $3",
)
.bind(kb_id)
.bind(limit)
diff --git a/crates/utopia-store/src/temporal.rs b/crates/utopia-store/src/temporal.rs
index 236d60fe..05ee9087 100644
--- a/crates/utopia-store/src/temporal.rs
+++ b/crates/utopia-store/src/temporal.rs
@@ -552,7 +552,7 @@ pub async fn list_conflicts(
LEFT JOIN entities oo ON oo.id = fo.object_id
LEFT JOIN entities no_ ON no_.id = fn_.object_id
WHERE c.kb_id = $1 AND c.status = 'open'
- ORDER BY c.created_at DESC
+ ORDER BY c.created_at DESC, c.id DESC
LIMIT $2 OFFSET $3",
)
.bind(kb_id)
diff --git a/crates/utopia-store/tests/a_page_never_skips_a_row.rs b/crates/utopia-store/tests/a_page_never_skips_a_row.rs
new file mode 100644
index 00000000..03977478
--- /dev/null
+++ b/crates/utopia-store/tests/a_page_never_skips_a_row.rs
@@ -0,0 +1,224 @@
+//! 分页翻完,每一行恰好出现一次(#646),打在真库上。
+//!
+//! Review 的各档与台账都是 `LIMIT/OFFSET` 分页。排序键若不唯一,并列的行每次查询的
+//! 先后可以不同——翻到第二页时第一页见过的又出来一次,另一行一页也没出现。时间戳
+//! 作排序键时,逐条写入的行很少并列;**一个事务里写下的行全都并列**(`now()` 在事务
+//! 里是同一个值),而 Review 的队列正是成批写的。量过:一次检查插下 572 个环,按 200 条
+//! 一页翻,翻出 557 个。
+//!
+//! 三份成批写下的数据,各翻一遍:
+//!
+//! 1. **公理违规**:一次 `reasoning::run` 插下几百个环,`open_violations` 翻
+//! 2. **本体缺陷**:一个语句插下几百行,`open_defects` 翻
+//! 3. **审核台账**:一个语句写下几百条审计事件,`review_history` 翻
+//!
+//! 断言的不只是「每行一次」,而是**翻出来的序列恰好是按时间倒序、再按 id 倒序排好的
+//! 全部行**。只断言每行一次是靠运气的:并列行的先后取决于执行计划,旧的排序在这份
+//! 数据上三次里只错一次。行的 id 是随机的,与写入的物理顺序无关,所以只要排序在并列
+//! 处没有定下来,序列就对不上——每次都对不上。
+//!
+//! 页长取质数(37),页边界落不到整齐的地方。没有 `UTOPIA_DATABASE_URL` 时跳过而不是
+//! 失败。自建自拆,绝不碰已有的库。
+
+use sqlx::PgPool;
+use std::future::Future;
+use utopia_store::reasoning;
+use uuid::Uuid;
+
+const PAGE: i64 = 37;
+
+struct Fixture {
+ org: Uuid,
+ kb: Uuid,
+}
+
+async fn seed(pool: &PgPool) -> anyhow::Result<Fixture> {
+ let (org, ws, kb) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7());
+ sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, 'paging-test')")
+ .bind(org)
+ .execute(pool)
+ .await?;
+ sqlx::query("INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, 'paging-test')")
+ .bind(ws)
+ .bind(org)
+ .execute(pool)
+ .await?;
+ sqlx::query(
+ "INSERT INTO knowledge_bases (id, workspace_id, name) VALUES ($1, $2, 'paging-test')",
+ )
+ .bind(kb)
+ .bind(ws)
+ .execute(pool)
+ .await?;
+ Ok(Fixture { org, kb })
+}
+
+/// 一页一页翻到底,按翻到的先后接起来
+async fn page_through<F, Fut>(mut page: F) -> anyhow::Result<Vec<Uuid>>
+where
+ F: FnMut(i64) -> Fut,
+ Fut: Future<Output = anyhow::Result<Vec<Uuid>>>,
+{
+ let mut seen = Vec::new();
+ let mut offset = 0;
+ loop {
+ let ids = page(offset).await?;
+ let n = ids.len() as i64;
+ seen.extend(ids);
+ if n < PAGE {
+ return Ok(seen);
+ }
+ offset += PAGE;
+ }
+}
+
+/// 翻出来的就是全部行,一行一次,并且时间戳并列时按 id 倒序——排序是全序
+async fn assert_total_order(
+ pool: &PgPool,
+ what: &str,
+ seen: &[Uuid],
+ all_ids: &str,
+ expected: usize,
+) -> anyhow::Result<()> {
+ let mut want: Vec<Uuid> = sqlx::query_scalar(all_ids).fetch_all(pool).await?;
+ assert_eq!(want.len(), expected, "{what}: 写下的行数");
+ want.sort_by(|a, b| b.cmp(a));
+ let mut distinct = seen.to_vec();
+ distinct.sort();
+ distinct.dedup();
+ assert_eq!(
+ distinct.len(),
+ seen.len(),
+ "{what}: {} 行翻到了不止一次",
+ seen.len() - distinct.len()
+ );
+ assert_eq!(seen.len(), expected, "{what}: 翻完只见到 {} 行", seen.len());
+ assert!(
+ seen == want,
+ "{what}: 并列的行没有按 id 定下先后,翻页之间会漂"
+ );
+ Ok(())
+}
+
+#[tokio::test]
+async fn paging_through_a_batch_sees_every_row_exactly_once() -> anyhow::Result<()> {
+ let Some(url) = utopia_store::test_db::url() else {
+ return Ok(());
+ };
+ let pool = PgPool::connect(&url).await?;
+ let f = seed(&pool).await?;
+
+ let result = async {
+ // ---- 一、公理违规:一个中心节点与 300 个节点两两互指,恒常谓词上就是 300 个环,
+ // 一次检查、一个事务插下
+ const CYCLES: usize = 300;
+ let (etype, part_of) = (Uuid::now_v7(), Uuid::now_v7());
+ sqlx::query("INSERT INTO entity_types (id, kb_id, key, label) VALUES ($1, $2, 't', 'T')")
+ .bind(etype)
+ .bind(f.kb)
+ .execute(&pool)
+ .await?;
+ sqlx::query(
+ "INSERT INTO relation_types (id, kb_id, key, label, temporal, is_transitive)
+ VALUES ($1, $2, 'part_of', 'part_of', 'eternal', TRUE)",
+ )
+ .bind(part_of)
+ .bind(f.kb)
+ .execute(&pool)
+ .await?;
+ sqlx::query(
+ "WITH n AS (
+ INSERT INTO entities (id, kb_id, type_id, canonical_name)
+ SELECT gen_random_uuid(), $1, $2, 'N' || i FROM generate_series(0, $4) AS i
+ RETURNING id, canonical_name
+ ), hub AS (SELECT id FROM n WHERE canonical_name = 'N0'),
+ spokes AS (SELECT id FROM n WHERE canonical_name <> 'N0')
+ INSERT INTO facts (id, kb_id, subject_id, predicate_id, object_id)
+ SELECT gen_random_uuid(), $1, hub.id, $3, spokes.id FROM hub, spokes
+ UNION ALL
+ SELECT gen_random_uuid(), $1, spokes.id, $3, hub.id FROM hub, spokes",
+ )
+ .bind(f.kb)
+ .bind(etype)
+ .bind(part_of)
+ .bind(CYCLES as i32)
+ .execute(&pool)
+ .await?;
+ let r = reasoning::run(&pool, f.kb).await?;
+ assert_eq!(r.inserted, CYCLES, "一次检查插下这些环");
+ let seen = page_through(|offset| {
+ let pool = pool.clone();
+ async move {
+ Ok(reasoning::open_violations(&pool, f.kb, PAGE, offset)
+ .await?
+ .into_iter()
+ .map(|v| v.id)
+ .collect())
+ }
+ })
+ .await?;
+ let ids = format!("SELECT id FROM axiom_violations WHERE kb_id = '{}'", f.kb);
+ assert_total_order(&pool, "公理违规", &seen, &ids, CYCLES).await?;
+
+ // ---- 二、本体缺陷:一个语句插下 250 行,detected_at 全相同
+ const DEFECTS: usize = 250;
+ sqlx::query(
+ "INSERT INTO ontology_defects (id, kb_id, kind, subject, other)
+ SELECT gen_random_uuid(), $1, 'rules_disagree', gen_random_uuid(), gen_random_uuid()
+ FROM generate_series(1, $2)",
+ )
+ .bind(f.kb)
+ .bind(DEFECTS as i32)
+ .execute(&pool)
+ .await?;
+ let seen = page_through(|offset| {
+ let pool = pool.clone();
+ async move {
+ Ok(reasoning::open_defects(&pool, f.kb, PAGE, offset)
+ .await?
+ .into_iter()
+ .map(|d| d.id)
+ .collect())
+ }
+ })
+ .await?;
+ let ids = format!("SELECT id FROM ontology_defects WHERE kb_id = '{}'", f.kb);
+ assert_total_order(&pool, "本体缺陷", &seen, &ids, DEFECTS).await?;
+
+ // ---- 三、审核台账:一个语句写下 250 条,created_at 全相同
+ const EVENTS: usize = 250;
+ sqlx::query(
+ "INSERT INTO audit_events (id, kb_id, action, target_kind)
+ SELECT gen_random_uuid(), $1, 'review.decided', 'violation'
+ FROM generate_series(1, $2)",
+ )
+ .bind(f.kb)
+ .bind(EVENTS as i32)
+ .execute(&pool)
+ .await?;
+ let seen = page_through(|offset| {
+ let pool = pool.clone();
+ async move {
+ Ok(
+ utopia_store::audit::review_history(&pool, f.kb, PAGE, offset)
+ .await?
+ .0
+ .into_iter()
+ .map(|e| e.id)
+ .collect(),
+ )
+ }
+ })
+ .await?;
+ let ids = format!("SELECT id FROM audit_events WHERE kb_id = '{}'", f.kb);
+ assert_total_order(&pool, "审核台账", &seen, &ids, EVENTS).await?;
+ Ok::<_, anyhow::Error>(())
+ }
+ .await;
+ // 审计事件只许追加(删不掉),留在测试库里;它们挂在这个自建的库名下,不串到别处
+ sqlx::query("DELETE FROM organizations WHERE id = $1")
+ .bind(f.org)
+ .execute(&pool)
+ .await?;
+ result
+}