Skip to content

World's first open-source enterprise world model.

Queue phrase alignment after the last kind-word round, and let it wait while one is pending (#926)

WaylandYangb3919ca8b4bf
diff --git a/crates/utopia-server/src/phrase_alignment.rs b/crates/utopia-server/src/phrase_alignment.rs
index 0047b951..2a81b7e2 100644
--- a/crates/utopia-server/src/phrase_alignment.rs
+++ b/crates/utopia-server/src/phrase_alignment.rs
@@ -361,6 +361,25 @@ pub async fn align_phrases_reasking(
// 对齐是判断题:让模型按端点默认的强度想,不用工作区给抽取设的 minimal
let client = llm_util::chat_client_thinking(&settings)
.ok_or_else(|| anyhow::anyhow!("Chat model not configured; cannot align phrases"))?;
+ // 类别词对齐还排着或跑着:两端的类还没定,现在判的签名指纹马上就变,判了也是重判。
+ // 等它收尾再来——排一份半分钟后的,排着的至多一份。别的入口(本体页、审核页、每篇文档
+ // 抽完)排的短语对齐也从这里过,所以这一道守住就够
+ if utopia_store::jobs::pending_for_kb(pool, "align_types", kb_id).await? {
+ tracing::info!(%kb_id, "短语对齐:类别词对齐还没收尾,半分钟后再看");
+ let payload = if reask == 0 {
+ serde_json::json!({ "kb_id": kb_id })
+ } else {
+ serde_json::json!({ "kb_id": kb_id, "reask": reask })
+ };
+ utopia_store::jobs::enqueue_unless_queued_after(
+ pool,
+ "align_phrases",
+ payload,
+ std::time::Duration::from_secs(30),
+ )
+ .await?;
+ return Ok(());
+ }
// 一个库同时只跑一份,理由同类别词对齐(并行跑会把端点打出 502)
let mut guard = pool.acquire().await?;
let locked: bool =
diff --git a/crates/utopia-server/src/phrase_alignment_tests.rs b/crates/utopia-server/src/phrase_alignment_tests.rs
index 5fc6626a..4da815fc 100644
--- a/crates/utopia-server/src/phrase_alignment_tests.rs
+++ b/crates/utopia-server/src/phrase_alignment_tests.rs
@@ -223,6 +223,57 @@ fn none() -> Vec<Value> {
vec![vote(None, None), vote(None, None), nothing_implied()]
}
+/// 类别词对齐还排着时短语对齐不动手:两端的类还没定,判了也是重判。它给自己排一份半分钟
+/// 后的,类别词对齐收尾后照常判
+#[tokio::test]
+async fn phrase_alignment_waits_while_a_kind_word_round_is_pending() -> anyhow::Result<()> {
+ let Some(f) = Fx::new().await? else {
+ return Ok(());
+ };
+ let run = async {
+ f.script(bound());
+ sqlx::query("INSERT INTO jobs (kind, payload) VALUES ('align_types', $1)")
+ .bind(json!({ "kb_id": f.kb }))
+ .execute(&f.pool)
+ .await?;
+ f.run().await?;
+ assert_eq!(
+ f.requests().len(),
+ 0,
+ "nothing is asked while kind words are still being decided"
+ );
+ let deferred: Vec<(String, bool)> = sqlx::query_as(
+ "SELECT status, run_at > now() FROM jobs
+ WHERE kind='align_phrases' AND payload->>'kb_id'=$1",
+ )
+ .bind(f.kb.to_string())
+ .fetch_all(&f.pool)
+ .await?;
+ assert_eq!(
+ deferred,
+ vec![("queued".to_string(), true)],
+ "one phrase round is queued for later"
+ );
+ f.run().await?;
+ assert_eq!(
+ f.requests().len(),
+ 0,
+ "still waiting: the kind-word job is still queued"
+ );
+ f.clear_jobs().await?;
+ f.run().await?;
+ assert_eq!(
+ f.requests().len(),
+ 2,
+ "the kind-word round gone, the two votes are asked"
+ );
+ anyhow::Ok(())
+ }
+ .await;
+ f.cleanup().await?;
+ run
+}
+
#[tokio::test]
async fn a_property_declared_on_an_ancestor_is_offered_with_its_basis_and_bound(
) -> anyhow::Result<()> {
diff --git a/crates/utopia-server/src/type_alignment.rs b/crates/utopia-server/src/type_alignment.rs
index 16539677..50f59149 100644
--- a/crates/utopia-server/src/type_alignment.rs
+++ b/crates/utopia-server/src/type_alignment.rs
@@ -371,6 +371,11 @@ async fn align_types_locked(
// 每次多等一会。从前只有失败的批次会再排,漏答的词就只能等下一篇文档来排——最后
// 一篇之后没有下一篇,它们就永远没有结论;而漏答不写任何行,本体页也看不见
let unfinished = failed > 0 || unanswered > 0;
+ // 还有下一轮(来了新词、有过期的,或没判完要再问)就先不排短语对齐:两端的类还在变,
+ // 这时判的签名指纹一变就得重判——100 篇的一次跑里,短语对齐第一轮判的 1010 条签名
+ // 里有 713 条因为类别词第二轮改了类而重判。最后一轮(不再排自己)才排它;再问的
+ // 次数用尽那一支也排:判定不会再变了
+ let mut another_round = false;
if changed {
utopia_store::jobs::enqueue_unless_queued(
pool,
@@ -378,6 +383,7 @@ async fn align_types_locked(
serde_json::json!({ "kb_id": kb_id }),
)
.await?;
+ another_round = true;
} else if unfinished && reask < MAX_REASK {
let delay = std::time::Duration::from_secs(20 * u64::from(reask + 1));
tracing::info!(%kb_id, failed, unanswered, reask = reask + 1, delay_secs = delay.as_secs(), "类别词对齐没判完,稍后再问");
@@ -388,16 +394,19 @@ async fn align_types_locked(
delay,
)
.await?;
+ another_round = true;
} else if unfinished {
tracing::warn!(%kb_id, failed, unanswered, reask, "类别词对齐问了几轮仍没判完,等下一篇文档或本体改动再问");
}
- // 两端的类定了,短语的签名才定:短语对齐排在它后面
- utopia_store::jobs::enqueue_unless_queued(
- pool,
- "align_phrases",
- serde_json::json!({ "kb_id": kb_id }),
- )
- .await?;
+ // 两端的类定了,短语的签名才定:短语对齐排在类别词对齐**收尾**之后
+ if !another_round {
+ utopia_store::jobs::enqueue_unless_queued(
+ pool,
+ "align_phrases",
+ serde_json::json!({ "kb_id": kb_id }),
+ )
+ .await?;
+ }
Ok(())
}
diff --git a/crates/utopia-server/src/type_alignment_tests.rs b/crates/utopia-server/src/type_alignment_tests.rs
index 17c3f63a..e75e9ecd 100644
--- a/crates/utopia-server/src/type_alignment_tests.rs
+++ b/crates/utopia-server/src/type_alignment_tests.rs
@@ -692,6 +692,13 @@ async fn an_unreadable_reply_is_asked_again_a_bounded_number_of_times() -> anyho
.bind(f.kb.to_string())
.fetch_all(&f.pool)
.await?;
+ // 还要再问一轮:短语对齐先不排,两端的类还没定
+ let phrases_queued_early: i64 = sqlx::query_scalar(
+ "SELECT count(*) FROM jobs WHERE kind='align_phrases' AND payload->>'kb_id'=$1",
+ )
+ .bind(f.kb.to_string())
+ .fetch_one(&f.pool)
+ .await?;
sqlx::query("DELETE FROM jobs WHERE payload->>'kb_id'=$1")
.bind(f.kb.to_string())
.execute(&f.pool)
@@ -704,6 +711,13 @@ async fn an_unreadable_reply_is_asked_again_a_bounded_number_of_times() -> anyho
.bind(f.kb.to_string())
.fetch_one(&f.pool)
.await?;
+ // 这是最后一轮:短语对齐现在排
+ let phrases_queued_late: i64 = sqlx::query_scalar(
+ "SELECT count(*) FROM jobs WHERE kind='align_phrases' AND payload->>'kb_id'=$1",
+ )
+ .bind(f.kb.to_string())
+ .fetch_one(&f.pool)
+ .await?;
let count = f.requests().len();
f.cleanup().await?;
assert_eq!(count, 4, "两轮各问两票");
@@ -714,5 +728,7 @@ async fn an_unreadable_reply_is_asked_again_a_bounded_number_of_times() -> anyho
"第一轮之后自己排一份 reask=1"
);
assert_eq!(after, 0, "排够次数就不再排");
+ assert_eq!(phrases_queued_early, 0, "还要再问时短语对齐不排");
+ assert_eq!(phrases_queued_late, 1, "最后一轮排短语对齐");
Ok(())
}
diff --git a/crates/utopia-store/src/jobs.rs b/crates/utopia-store/src/jobs.rs
index cfcc9f1c..e7ae331a 100644
--- a/crates/utopia-store/src/jobs.rs
+++ b/crates/utopia-store/src/jobs.rs
@@ -113,6 +113,20 @@ pub async fn enqueue_unless_queued_after(
Ok(row.map(|(id,)| id))
}
+/// 这个库有没有这一种任务排着或跑着(按 payload 里的 kb_id 看,不比整份 payload:
+/// 同一种任务的 payload 可能多带一个 reask)
+pub async fn pending_for_kb(pool: &PgPool, kind: &str, kb_id: Uuid) -> AppResult<bool> {
+ Ok(sqlx::query_scalar(
+ "SELECT EXISTS (SELECT 1 FROM jobs
+ WHERE kind = $1 AND status IN ('queued', 'running')
+ AND payload->>'kb_id' = $2)",
+ )
+ .bind(kind)
+ .bind(kb_id.to_string())
+ .fetch_one(pool)
+ .await?)
+}
+
pub async fn enqueue(pool: &PgPool, kind: &str, payload: serde_json::Value) -> AppResult<i64> {
enqueue_with_max_attempts(pool, kind, payload, 3).await
}
diff --git a/scripts/bench/README.md b/scripts/bench/README.md
index cfd3dd5b..1bdec573 100644
--- a/scripts/bench/README.md
+++ b/scripts/bench/README.md
@@ -691,3 +691,35 @@ bound / none / undecided。0044 的门槛:裁判精度不低于完整原型的
第一批判错的"无"第二批照样错——温库就是这样工作的。勘误在温库上复查了全部 200 篇:新绑定给旧文档也添了
类型化事实,所以旧文档也在它的清单里,每篇边际里的勘误那 0.4 万有一半是复查旧文档。
+### 第五次真跑:短语对齐排在类别词对齐收尾之后(2026-09-25 夜,#926,两组 + dev 对照一组)
+
+组 3 的日志里短语对齐判了 1847 次而签名只有约 1000 条:第二轮类别词对齐(562 个词)和第一轮短语对齐
+同时在跑,类别词绑完改了实体的类,签名指纹全变,713 条重判。#926 让类别词对齐只在最后一轮才排短语
+对齐,短语对齐开跑前若发现类别词任务还排着就推迟半分钟。同一语料、同一模型、同一裁判:
+
+| | 组 5(#926) | 组 6(#926) | 组 7(dev 对照,同一时段) |
+|---|---|---|---|
+| 签名 bound / none / undecided | 419 / 362 / 165 | 414 / 377 / 166 | 420 / 1405 / 165 |
+| 类型化 勘误前 → 后 | 739 → | 694 → | 707 → |
+| gold recall 勘误前(同句) | 9.4%(14.2%) | 8.7%(13.4%) | 9.4%(14.2%) |
+| gold recall 勘误后(同句) | 12.4%(17.5%) | 12.1%(17.5%) | — |
+| 裁判精度 勘误前,200 抽样 | 75.5% | 70.6% | 78.5% |
+| 裁判精度 勘误后,200 抽样 | 78.5% | 75.0% | 74.0% |
+| **裁判精度 勘误后,600 抽样(种子 7)** | **77.2%** | **81.8%** | **82.0%**(组 4 同口径 81.7%) |
+| 每篇 token | 3.1 万 | 2.9 万 | 3.2 万 |
+
+读数:
+- 签名干净了:dev 上判"无"的 1405 条里约 1000 条是类别词还在变时生成的过渡签名,#926 之后没有了;
+ 短语判定从 1847 次回到 980 次。召回不变。
+- token 几乎没省(每篇 3.1 / 2.9 万对 3.2 万):被重判的签名便宜。对齐的大头是提规则——组 5 问了 1506 次
+ 提出 101 条(类别词 621 问 32 提,短语 411 问 69 提)——和两票本身。
+- **200 条裁判太粗**:同一个库 200 条与 600 条能差十个点(组 6 的 70.6% 对 81.8%),裁判自己也是模型,
+ 两次判同一批也不一样。门槛 75.3% 与实际水平(80% 上下)只差五个点,200 条分不出来。`--judge` 缺省
+ 改成 600;这一节之前的精度都是 200 条的,读的时候按 ±5 个点看。
+- 600 条口径下 #926 与 dev 没有可分辨的差别(77.2 / 81.8 对 81.7 / 82.0)。
+
+按语句数过滤提规则**不该做**:提出的 101 条规则里 80 条来自单语句签名。类别词那 621 次询问命中 5%,
+是能砍的;条件第二票再砍一成多。三样都做每篇也在 2.3 万上下:对齐按签名计费,这份语料每篇 9 到 10 条
+新签名、每条约 1.4 千 token,这就是它的地板。0044 的 token 门槛在 Re-DocRED 上要么换口径(每条新签名
+的成本),要么换同领域语料再量。
+
diff --git a/scripts/bench/typed.mjs b/scripts/bench/typed.mjs
index f60d5def..fe43b369 100644
--- a/scripts/bench/typed.mjs
+++ b/scripts/bench/typed.mjs
@@ -17,7 +17,7 @@
//
// 用法:
// node scripts/bench/typed.mjs --label run1 # 完整一组
-// node scripts/bench/typed.mjs --label run1 --judge 200 # 加裁判抽样 200 条
+// node scripts/bench/typed.mjs --label run1 --judge 600 # 加裁判抽样 600 条(200 条时同一个库两次裁能差十个点)
// node scripts/bench/typed.mjs --kb <id> --score # 只对已有的库重新打分
// node scripts/bench/typed.mjs --label dry --dry-run # 建库、装本体、灌语料、等解析,不抽取:验管线
// node scripts/bench/typed.mjs --label run1 --judge 200 --errata # 对齐之后再跑勘误 agent,报前后两份分与撤错多少
@@ -490,7 +490,7 @@ if (args["approve-rules"]) rules = await approveRules(KB);
// 勘误前的分:已经跑过勘误的库也能按 scope 算回来(撤掉的算回来、加上的不算)
const result = score(KB, "before");
if (rules) result.rules = rules;
-if (args.judge) result.judge = await judge(KB, Number(args.judge) || 200, Number(args.seed || 1));
+if (args.judge) result.judge = await judge(KB, Number(args.judge) || 600, Number(args.seed || 1));
if (args.errata) {
// 勘误前的分留着,勘误后再打一次:0044 §7 的度量是两份分的差,与撤错了多少
result.before_errata = { gold_recall: result.gold_recall, gold_recall_same_sentence: result.gold_recall_same_sentence, typed_facts: result.typed_facts, judge: result.judge };
@@ -500,7 +500,7 @@ if (args.errata) {
const after = score(KB, "after");
result.after_errata = { gold_recall: after.gold_recall, gold_recall_same_sentence: after.gold_recall_same_sentence, typed_facts: after.typed_facts };
if (args.judge) {
- result.after_errata.judge = await judge(KB, Number(args.judge) || 200, Number(args.seed || 1));
+ result.after_errata.judge = await judge(KB, Number(args.judge) || 600, Number(args.seed || 1));
result.errata.removed = await judgeRetracted(KB);
}
}