Skip to content

World's first open-source enterprise world model.

The extraction progress counts each document once, not once per pipeline (#715)

Qishang Zhong00e684f949a4
diff --git a/crates/utopia-core/src/models.rs b/crates/utopia-core/src/models.rs
index 8dd446f5..ae1277f8 100644
--- a/crates/utopia-core/src/models.rs
+++ b/crates/utopia-core/src/models.rs
@@ -1597,14 +1597,17 @@ pub struct RelationAxioms {
/// 文库的一页,连同这一页之外的统计。
///
-/// **统计不受名字/状态筛选影响**:`ready` / `extracting` / `failed` 说的是这个
-/// 来源里有多少,那是批量按钮的作用范围,跟你此刻在搜什么无关。
+/// **统计不受名字/状态筛选影响**:`ready` / `done` / `extracting` / `failed` 说的是
+/// 这个来源里有多少,那是批量按钮的作用范围,跟你此刻在搜什么无关。
#[derive(Debug, Clone, Serialize)]
pub struct DocumentPage {
pub docs: Vec<Document>,
/// 命中筛选的总数(分页器用它)
pub total: i64,
+ /// 摄入完成(`status = 'ready'`)的篇数:重抽的作用范围
pub ready: i64,
+ /// 抽取完成(`graph_status = 'done'`)的篇数:抽取进度条的分子
+ pub done: i64,
pub extracting: i64,
pub failed: i64,
/// 整库的墓碑数(删了、没清的)——左栏「已删除」那一行的数字,不随作用域变
diff --git a/crates/utopia-store/src/documents.rs b/crates/utopia-store/src/documents.rs
index 6fb5e4fe..e0651030 100644
--- a/crates/utopia-store/src/documents.rs
+++ b/crates/utopia-store/src/documents.rs
@@ -476,10 +476,14 @@ pub async fn page(
.fetch_one(pool)
.await?;
- // 统计只按来源作用域算:那两个批量按钮作用于整个来源,不是你搜出来的那几条
- let stats: (i64, i64, i64) = sqlx::query_as(
+ // 统计只按来源作用域算:那两个批量按钮作用于整个来源,不是你搜出来的那几条。
+ // 四个数各对应一列:`ready` 是摄入(`status`),后三个是抽取(`graph_status`)。
+ // 两个维度不能相加——一篇「摄入已完成、图谱还在抽」的文档两边都占,进度条若拿
+ // `ready + extracting` 当分母,两篇文档会显示成 2 / 4
+ let stats: (i64, i64, i64, i64) = sqlx::query_as(
"SELECT
count(*) FILTER (WHERE status = 'ready'),
+ count(*) FILTER (WHERE graph_status = 'done'),
count(*) FILTER (WHERE graph_status IN ('queued', 'extracting')),
count(*) FILTER (WHERE graph_status = 'failed')
FROM documents
@@ -506,8 +510,9 @@ pub async fn page(
docs,
total,
ready: stats.0,
- extracting: stats.1,
- failed: stats.2,
+ done: stats.1,
+ extracting: stats.2,
+ failed: stats.3,
deleted: deleted_total,
})
}
diff --git a/crates/utopia-store/tests/extraction_progress_counts_a_document_once.rs b/crates/utopia-store/tests/extraction_progress_counts_a_document_once.rs
new file mode 100644
index 00000000..153f997a
--- /dev/null
+++ b/crates/utopia-store/tests/extraction_progress_counts_a_document_once.rs
@@ -0,0 +1,87 @@
+//! 文库的抽取进度:每篇文档只数一次(#715)。
+//!
+//! 进度条从前拿 `ready`(摄入完成)当分子、`ready + extracting` 当分母。一篇「摄入完了、
+//! 图谱还在抽」的文档在两个维度里各算一次:库里只有一篇正在抽的文档,也显示成 1 / 2。
+//! 现在分子分母都只看 `graph_status`:`done` 与 `extracting`。删掉的文档不算,
+//! 抽取失败的单独数。
+//!
+//! 没有 `UTOPIA_DATABASE_URL` 时跳过而不是失败。自建自拆,绝不碰已有的库。
+
+use sqlx::PgPool;
+use utopia_store::documents;
+use uuid::Uuid;
+
+async fn document(
+ pool: &PgPool,
+ kb: Uuid,
+ name: &str,
+ graph_status: &str,
+ deleted: bool,
+) -> anyhow::Result<()> {
+ sqlx::query(
+ "INSERT INTO documents (id, kb_id, filename, sha256, status, graph_status, deleted_at)
+ VALUES ($1, $2, $3, $3, 'ready', $4, CASE WHEN $5 THEN now() END)",
+ )
+ .bind(Uuid::now_v7())
+ .bind(kb)
+ .bind(name)
+ .bind(graph_status)
+ .bind(deleted)
+ .execute(pool)
+ .await?;
+ Ok(())
+}
+
+#[tokio::test]
+async fn a_document_still_extracting_is_not_counted_as_done() -> anyhow::Result<()> {
+ let Some(url) = utopia_store::test_db::url() else {
+ return Ok(());
+ };
+ let pool = PgPool::connect(&url).await?;
+ let (org, ws, kb) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7());
+ sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, 'progress-test')")
+ .bind(org)
+ .execute(&pool)
+ .await?;
+ sqlx::query("INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, 'progress-test')")
+ .bind(ws)
+ .bind(org)
+ .execute(&pool)
+ .await?;
+ sqlx::query(
+ "INSERT INTO knowledge_bases (id, workspace_id, name) VALUES ($1, $2, 'progress-test')",
+ )
+ .bind(kb)
+ .bind(ws)
+ .execute(&pool)
+ .await?;
+
+ let run = async {
+ document(&pool, kb, "done.md", "done", false).await?;
+ document(&pool, kb, "extracting.md", "extracting", false).await?;
+ document(&pool, kb, "queued.md", "queued", false).await?;
+ document(&pool, kb, "failed.md", "failed", false).await?;
+ // 删掉的文档哪一格都不算,包括它删之前正在抽的
+ document(&pool, kb, "deleted.md", "extracting", true).await?;
+ let page = documents::page(&pool, kb, None, None, None, false, 50, 0).await?;
+ // 摄入完成的四篇都是 ready——重抽的作用范围不变
+ assert_eq!(page.ready, 4);
+ assert_eq!(page.done, 1);
+ assert_eq!(page.extracting, 2, "queued 与 extracting 都算在抽");
+ assert_eq!(page.failed, 1);
+ // 进度条因此是 done / (done + extracting) = 1 / 3。旧的写法 ready / (ready + extracting)
+ // 会是 4 / 6:两篇还没抽完的、一篇抽失败的都被算成了完成
+ anyhow::Ok(())
+ }
+ .await;
+
+ let _ = sqlx::query("DELETE FROM knowledge_bases WHERE id = $1")
+ .bind(kb)
+ .execute(&pool)
+ .await;
+ let _ = sqlx::query("DELETE FROM organizations WHERE id = $1")
+ .bind(org)
+ .execute(&pool)
+ .await;
+ run
+}
diff --git a/web/src/api.ts b/web/src/api.ts
index 498b96dc..69f56ab1 100644
--- a/web/src/api.ts
+++ b/web/src/api.ts
@@ -1658,9 +1658,11 @@ export const api = {
return request<{
docs: Doc[];
total: number;
- /** 下面三个**只按来源作用域算**,不受名字/状态筛选影响——
+ /** 下面四个**只按来源作用域算**,不受名字/状态筛选影响——
* 它们是批量按钮的作用范围 */
ready: number;
+ /** `graph_status = 'done'`:抽取进度条的分子 */
+ done: number;
extracting: number;
failed: number;
/** 整库的墓碑数(删了、没清的),不随作用域变 */
diff --git a/web/src/pages/Library.tsx b/web/src/pages/Library.tsx
index 3401c447..7c039370 100644
--- a/web/src/pages/Library.tsx
+++ b/web/src/pages/Library.tsx
@@ -670,12 +670,14 @@ export function Library() {
)}
{/* 抽取进度。**数来自服务端**,按来源作用域算——从前是数当前页里的,
- 翻一页进度条就跳 */}
+ 翻一页进度条就跳。分子分母都只走 `graph_status`:从前分子是
+ `status='ready'`(摄入完成),一篇「摄入已完成、图谱还在抽」的文档
+ 被两个维度各数一次,两篇文档能显示成 2 / 4 */}
{(() => {
const pending = docs.data?.extracting ?? 0;
if (pending === 0) return null;
- const total = (docs.data?.ready ?? 0) + pending;
- const done = total - pending;
+ const done = docs.data?.done ?? 0;
+ const total = done + pending;
return (
<div className="mb-3 glass rounded-panel px-4 py-3">
<div className="flex items-center justify-between text-small text-ink-2 mb-2">