Skip to content

World's first open-source enterprise world model.

Read the kind-word binding reply in the shapes the model actually writes so a whole base no longer stays untyped in silence (#894)

WaylandYangbd66d049503a
diff --git a/crates/utopia-extract/src/align.rs b/crates/utopia-extract/src/align.rs
index 9d067892..a9e4e33e 100644
--- a/crates/utopia-extract/src/align.rs
+++ b/crates/utopia-extract/src/align.rs
@@ -12,6 +12,13 @@
//! 整批;键不在那一项的候选里、id 不在批里、同一个 id 的第二次都算坏;截断的回复退到
//! 最后一个完整的对。没答到的 id 就是没答到——调用方按 id 对账,缺席是「再问」,
//! 不是 null。
+//!
+//! **形状也宽容**:模型(实测 DeepSeek-V3.2)并不总照样例写。它会把整段答成按 id
+//! 作键的对象(`{"0": null, "1": "organization"}`),会把键包进单元素数组
+//! (`{"0": ["organization"], "4": [null]}`),也会把 `b` 写成对象、把一对写成
+//! `{"id": 0, "key": ...}`、或者不要外层对象只给数组。这些说的都是同一件事,读法
+//! 只有一种;温度为零时同一段提示词回来的形状还是同一个,读不出就是每一轮都读不出——
+//! 一个库连着十二篇文档一个词都没绑上,就是这么来的。读不出的键与 id 算坏项,不当缺席。
use std::collections::{HashMap, HashSet};
@@ -157,8 +164,8 @@ pub fn parse_kind_word_response(
let mut choices = Vec::new();
let mut malformed = 0usize;
let mut seen = HashSet::new();
- for pair in pairs(&value) {
- match parse_pair(pair, &by_id) {
+ for (id, key) in answers(&value) {
+ match parse_pair(&id, &key, &by_id) {
Some(choice) if seen.insert(choice.id) => choices.push(choice),
_ => malformed += 1,
}
@@ -168,14 +175,22 @@ pub fn parse_kind_word_response(
/// 先按常规取块(第一个 `{` 到最后一个 `}`);解不开才从第一个 `{` 取到结尾去修补。
/// 取块与修补的分工同 `open.rs`:紧凑回复里 `}` 只在结尾出现,截断的回复要么没有 `}`,
-/// 要么最后一个 `}` 不是结尾
+/// 要么最后一个 `}` 不是结尾。回复不要外层对象、直接给数组的,取第一个 `[` 到最后
+/// 一个 `]`——只在文字本身以 `[` 开头时这么读,别把对象里的一对当成整段
pub(crate) fn parse_value(raw: &str) -> anyhow::Result<Value> {
+ let text = json_text(raw).trim();
+ if text.starts_with('[') {
+ if let Some(end) = text.rfind(']') {
+ if let Ok(v) = serde_json::from_str::<Value>(&text[..=end]) {
+ return Ok(v);
+ }
+ }
+ }
let block = json_block(raw)
.and_then(|b| serde_json::from_str::<Value>(&b).map_err(anyhow::Error::from));
match block {
Ok(v) => Ok(v),
Err(e) => {
- let text = json_text(raw);
let fixed = text
.find('{')
.map(|s| &text[s..])
@@ -187,23 +202,49 @@ pub(crate) fn parse_value(raw: &str) -> anyhow::Result<Value> {
}
}
-/// 顶层 `b` 下的数组;缺了或不是数组就当空——那不是坏项,是一项都没答
-fn pairs(value: &Value) -> &[Value] {
- value
- .get("b")
- .and_then(Value::as_array)
- .map_or(&[], Vec::as_slice)
+/// 回复里的每一条答案:(id,键)两个原始值,形状还没验。
+///
+/// 认四种写法,说的都是「这个 id 选了这个键」:`{"b": [[id, key]]}`(样例)、
+/// `{"b": {"id": key}}`、没有 `b` 的顶层对象 `{"id": key}`、顶层数组 `[[id, key]]`;
+/// 数组里的一条也可以是 `{"id": .., "key": ..}`。`b` 在就只看 `b`,顶层别的键是
+/// 模型的旁白,不是答案。缺了 `b` 又不是对象或数组的,一条都没有
+fn answers(value: &Value) -> Vec<(Value, Value)> {
+ let listed = value.get("b").unwrap_or(value);
+ match listed {
+ Value::Array(entries) => entries
+ .iter()
+ .map(|entry| match entry {
+ Value::Array(pair) if pair.len() >= 2 => (pair[0].clone(), pair[1].clone()),
+ Value::Object(map) => (
+ map.get("id").cloned().unwrap_or(Value::Null),
+ map.get("key")
+ .or_else(|| map.get("class"))
+ .or_else(|| map.get("b"))
+ .cloned()
+ .unwrap_or(Value::Null),
+ ),
+ other => (other.clone(), Value::Null),
+ })
+ .collect(),
+ Value::Object(map) => map
+ .iter()
+ .map(|(k, v)| (Value::String(k.clone()), v.clone()))
+ .collect(),
+ _ => Vec::new(),
+ }
}
-/// `[id, key | null]`
-fn parse_pair(v: &Value, by_id: &HashMap<i64, &KindWordItem<'_>>) -> Option<KindWordChoice> {
- let arr = v.as_array()?;
- if arr.len() < 2 {
- return None;
- }
- let id = item_id(&arr[0])?;
+/// 一条答案:id 得是这批里的,键得是 null 或候选里的一个(数组里包着的也算,
+/// `["organization"]`、`[null]`、`["organization", "机构 — 定义"]` 是模型的另几种写法,
+/// 不是另一种答案)
+fn parse_pair(
+ id: &Value,
+ key: &Value,
+ by_id: &HashMap<i64, &KindWordItem<'_>>,
+) -> Option<KindWordChoice> {
+ let id = item_id(id)?;
let item = by_id.get(&id)?;
- let key = match &arr[1] {
+ let key = match unwrapped(item, id, key)? {
Value::Null => None,
Value::String(written) => Some(candidate_key(item, written)?),
_ => return None,
@@ -211,6 +252,28 @@ fn parse_pair(v: &Value, by_id: &HashMap<i64, &KindWordItem<'_>>) -> Option<Kind
Some(KindWordChoice { id, key })
}
+/// 数组里的第一个值:模型会把键包进数组,后面有时跟着那个候选的标签与定义
+/// (`["organization", "机构 — 研究院、研究所……"]`),前面有时又抄一遍 id
+/// (`{"0": [0, "organization"]}`)——抄的 id 跳过。后面跟的若是**另一个候选的键**,
+/// 那是两个答案,不是一个——None,调用方算坏。空数组也是 None;不是数组的原样
+fn unwrapped<'v>(item: &KindWordItem<'_>, id: i64, v: &'v Value) -> Option<&'v Value> {
+ match v {
+ Value::Array(list) => {
+ let list = match list.first() {
+ Some(first) if item_id(first) == Some(id) => &list[1..],
+ _ => &list[..],
+ };
+ let (first, rest) = list.split_first()?;
+ let another_key = rest
+ .iter()
+ .filter_map(Value::as_str)
+ .any(|s| candidate_key(item, s).is_some());
+ (!another_key).then_some(first)
+ }
+ other => Some(other),
+ }
+}
+
/// id 是 JSON 整数;模型偶尔把它写成字符串,照数字读
pub(crate) fn item_id(v: &Value) -> Option<i64> {
match v {
@@ -405,7 +468,8 @@ mod tests {
assert!(parse_kind_word_response("no json here", &items).is_err());
}
- /// 没有 `b` 或不是数组:一项都没答,不是坏项
+ /// `b` 是空的:一项都没答,不是坏项。没有 `b`、键又不是 id 的:那是读不出的答案,
+ /// 算坏项——从前这种回复算「一项都没答」,一个库十几轮下来什么痕迹都不留
#[test]
fn a_reply_without_pairs_gives_no_choices() {
let f = Fixture::new();
@@ -413,9 +477,94 @@ mod tests {
let (choices, malformed) = parse_kind_word_response(r#"{"b": {}}"#, &items).unwrap();
assert!(choices.is_empty());
assert_eq!(malformed, 0);
+ let (choices, malformed) = parse_kind_word_response(r#"{"b": []}"#, &items).unwrap();
+ assert!(choices.is_empty());
+ assert_eq!(malformed, 0);
let (choices, malformed) = parse_kind_word_response(r#"{"x": 1}"#, &items).unwrap();
assert!(choices.is_empty());
+ assert_eq!(malformed, 1);
+ }
+
+ /// DeepSeek-V3.2 实测的写法:整段是按 id 作键的对象,没有 `b`。这就是 bench 里
+ /// 「12 篇文档 type_id 全空」的回复——它解出来必须和样例形状一模一样
+ #[test]
+ fn an_id_keyed_object_is_read_as_the_pair_list() {
+ let f = Fixture::new();
+ let items = f.items();
+ let raw = r#"{ "12": "organization", "13": null, "14": "structure" }"#;
+ let (choices, malformed) = parse_kind_word_response(raw, &items).unwrap();
+ assert_eq!(malformed, 0);
+ assert_eq!(
+ choices,
+ vec![
+ choice(12, Some("organization")),
+ choice(13, None),
+ choice(14, Some("structure")),
+ ]
+ );
+ // `b` 下面是对象而不是数组:一样读
+ let raw = r#"{"b": {"12": "organization", "13": null}}"#;
+ let (choices, malformed) = parse_kind_word_response(raw, &items).unwrap();
+ assert_eq!(malformed, 0);
+ assert_eq!(choices.len(), 2);
+ }
+
+ /// 同一个模型的另一种写法:键包在单元素数组里。空数组与两个键不是一个答案
+ #[test]
+ fn a_key_wrapped_in_a_one_element_array_is_unwrapped() {
+ let f = Fixture::new();
+ let items = f.items();
+ let raw = r#"{ "12": ["organization"], "13": [null], "14": [] }"#;
+ let (choices, malformed) = parse_kind_word_response(raw, &items).unwrap();
+ assert_eq!(malformed, 1, "空数组不是一个答案");
+ assert_eq!(
+ choices,
+ vec![choice(12, Some("organization")), choice(13, None)]
+ );
+ let raw = r#"{"b": [[12, ["organization", "person"]]]}"#;
+ let (choices, malformed) = parse_kind_word_response(raw, &items).unwrap();
+ assert_eq!(malformed, 1, "两个键不是一个答案");
+ assert!(choices.is_empty());
+ // 值里先抄一遍 id 再给键:`{"0": [0, "organization"]}`
+ let raw = r#"{"12": [12, "organization"], "13": [13, null]}"#;
+ let (choices, malformed) = parse_kind_word_response(raw, &items).unwrap();
+ assert_eq!(malformed, 0);
+ assert_eq!(
+ choices,
+ vec![choice(12, Some("organization")), choice(13, None)]
+ );
+ // 键后面跟着候选行(标签与定义):那是抄了一遍候选,不是第二个答案
+ let raw = r#"{"12": ["organization", "Organization — An organized group of people"], "13": [null, "no candidate fits"]}"#;
+ let (choices, malformed) = parse_kind_word_response(raw, &items).unwrap();
assert_eq!(malformed, 0);
+ assert_eq!(
+ choices,
+ vec![choice(12, Some("organization")), choice(13, None)]
+ );
+ }
+
+ /// 一对写成对象、或者整段不要外层对象只给数组:照读
+ #[test]
+ fn object_pairs_and_a_bare_array_parse() {
+ let f = Fixture::new();
+ let items = f.items();
+ let raw = r#"{"b": [{"id": 12, "key": "organization"}, {"id": 13, "key": null}]}"#;
+ let (choices, malformed) = parse_kind_word_response(raw, &items).unwrap();
+ assert_eq!(malformed, 0);
+ assert_eq!(
+ choices,
+ vec![choice(12, Some("organization")), choice(13, None)]
+ );
+ let raw = "```json\n[[12, \"organization\"], [14, \"structure\"]]\n```";
+ let (choices, malformed) = parse_kind_word_response(raw, &items).unwrap();
+ assert_eq!(malformed, 0);
+ assert_eq!(
+ choices,
+ vec![
+ choice(12, Some("organization")),
+ choice(14, Some("structure"))
+ ]
+ );
}
/// 同一个 id 答了两次:留第一次,第二次计入坏项
diff --git a/crates/utopia-server/src/main.rs b/crates/utopia-server/src/main.rs
index 21f9eb06..7db51c81 100644
--- a/crates/utopia-server/src/main.rs
+++ b/crates/utopia-server/src/main.rs
@@ -579,7 +579,14 @@ async fn dispatch(st: &state::AppState, job: &utopia_store::jobs::Job) -> anyhow
.and_then(|v| v.as_str())
.and_then(|s| s.parse().ok())
.ok_or_else(|| anyhow::anyhow!("payload 缺少 kb_id"))?;
- type_alignment::align_types(st, kb_id).await
+ // 自己再排的那份带着第几次;文档、建类排的没有这一项
+ let reask = job
+ .payload
+ .get("reask")
+ .and_then(|v| v.as_u64())
+ .and_then(|n| u32::try_from(n).ok())
+ .unwrap_or(0);
+ type_alignment::align_types_reasking(st, kb_id, reask).await
}
// 关系短语按签名绑到属性(0044 对齐的第二片):类别词绑完排一个,属性改了再排
"align_phrases" => {
diff --git a/crates/utopia-server/src/type_alignment.rs b/crates/utopia-server/src/type_alignment.rs
index eb77f381..21d5761b 100644
--- a/crates/utopia-server/src/type_alignment.rs
+++ b/crates/utopia-server/src/type_alignment.rs
@@ -71,8 +71,26 @@ fn agree(a: &Vote, b: &Vote) -> bool {
a == b
}
+/// 日志里放得下的一段回复:空白折成一个空格,最多这么多字符。
+const SNIPPET_CHARS: usize = 240;
+
+fn snippet(text: &str) -> String {
+ let flat = text.split_whitespace().collect::<Vec<_>>().join(" ");
+ let mut out: String = flat.chars().take(SNIPPET_CHARS).collect();
+ if flat.chars().count() > SNIPPET_CHARS {
+ out.push('…');
+ }
+ out
+}
+
+/// 一轮里没判完的(调用失败、回复读不出、模型漏答)自己再排几次;超过这个数就等
+/// 下一篇文档或本体的改动再问。不设上限的话,温度为零下一段每次都读不出的回复会让
+/// 任务每隔几十秒把同一段提示词再送一遍,没有尽头
+const MAX_REASK: u32 = 3;
+
/// 对一个库跑一遍:新出现的和过期的类别词各判一次,绑上的写类,没有的提成建议。
-pub async fn align_types(state: &AppState, kb_id: Uuid) -> anyhow::Result<()> {
+/// `reask` 是这份任务已经是第几次自己排的(文档、建类排的是 0)。
+pub async fn align_types_reasking(state: &AppState, kb_id: Uuid, reask: u32) -> anyhow::Result<()> {
let pool = &state.pool;
let kb = utopia_store::kbs::get(pool, kb_id).await?;
let settings = utopia_store::settings::get(pool, kb.workspace_id)
@@ -94,7 +112,7 @@ pub async fn align_types(state: &AppState, kb_id: Uuid) -> anyhow::Result<()> {
tracing::info!(%kb_id, "类别词对齐已有一份在跑,这次跳过");
return Ok(());
}
- let result = align_types_locked(state, kb_id, &settings, &client).await;
+ let result = align_types_locked(state, kb_id, reask, &settings, &client).await;
let _ = sqlx::query("SELECT pg_advisory_unlock(hashtext('align_types'), hashtext($1))")
.bind(kb_id.to_string())
.execute(&mut *guard)
@@ -105,6 +123,7 @@ pub async fn align_types(state: &AppState, kb_id: Uuid) -> anyhow::Result<()> {
async fn align_types_locked(
state: &AppState,
kb_id: Uuid,
+ reask: u32,
settings: &utopia_core::models::LlmSettings,
client: &utopia_llm::LlmClient,
) -> anyhow::Result<()> {
@@ -161,6 +180,8 @@ async fn align_types_locked(
let (mut bound, mut none, mut undecided, mut skipped) = (0usize, 0usize, 0usize, 0usize);
// 调用或解析失败的批次:这轮跳过,结束时自己再排一次,不等下一篇文档来排
let mut failed = 0usize;
+ // 问了、模型也答了、却没答到的词:两票缺一票就不下结论
+ let mut unanswered = 0usize;
for batch in todo.chunks(BATCH) {
let cands = candidates_for(state, kb_id, batch, &classes).await?;
// 两票:第二票把候选倒过来给,防止「选第一个」这种顺序偏好冒充一致
@@ -218,6 +239,23 @@ async fn align_types_locked(
}
};
skipped += malformed;
+ if choices.is_empty() {
+ // 解出来了却一对都没读到:回复的形状不是我们认得的。这和解析失败是一回事,
+ // 按失败算、留到下次。从前这里什么都不说,一个库连着十几轮一个词都没绑上,
+ // 日志里只有一串「完成 bound=0」——回复的开头要进日志,下次才知道它长什么样
+ tracing::warn!(
+ %kb_id,
+ pass,
+ items = items.len(),
+ malformed,
+ finish_reason = ?reply.finish_reason,
+ chars = reply.text.chars().count(),
+ reply = %snippet(&reply.text),
+ "类别词对齐回复读不出一项,这一批留到下次"
+ );
+ failed += 1;
+ continue;
+ }
for c in choices {
let Ok(i) = usize::try_from(c.id) else {
continue;
@@ -242,6 +280,7 @@ async fn align_types_locked(
let (ans_a, ans_b) = answered[i];
if !ans_a || !ans_b {
// 有一票没答到:不下结论,下次再问
+ unanswered += 1;
continue;
}
let record = serde_json::json!({ "first": a, "second": b });
@@ -305,13 +344,16 @@ async fn align_types_locked(
}
}
}
- tracing::info!(%kb_id, bound, none, undecided, skipped, failed, "类别词对齐完成");
+ tracing::info!(%kb_id, bound, none, undecided, skipped, failed, unanswered, "类别词对齐完成");
+ if unanswered > 0 {
+ tracing::warn!(%kb_id, unanswered, "类别词对齐有词模型没答到,这些词这轮没有结论");
+ }
if bound + none + undecided > 0 {
state.emit_graph(kb_id);
}
- // 同短语对齐:失败过、来了没试过的新词、本轮判完的又过期了,就再排一次
- // 同短语对齐:「过期」不限本轮判的,跑着时建的类也要让老绑定再判一次
- let again = failed > 0 || {
+ // 同短语对齐:来了没试过的新词、本轮判完的又过期了,就再排一次(从头算一份,
+ // 新词换了提示词)。「过期」不限本轮判的,跑着时建的类也要让老绑定再判一次
+ let changed = {
let stale_now: HashSet<String> = type_bindings::stale(pool, kb_id)
.await?
.into_iter()
@@ -325,13 +367,29 @@ async fn align_types_locked(
.iter()
.any(|b| b.decided_by != "person" && stale_now.contains(&b.kind_word))
};
- if again {
+ // 本轮没判完的(调用失败、回复读不出、模型漏答了几个 id)自己再排,最多 MAX_REASK 次,
+ // 每次多等一会。从前只有失败的批次会再排,漏答的词就只能等下一篇文档来排——最后
+ // 一篇之后没有下一篇,它们就永远没有结论;而漏答不写任何行,本体页也看不见
+ let unfinished = failed > 0 || unanswered > 0;
+ if changed {
utopia_store::jobs::enqueue_unless_queued(
pool,
"align_types",
serde_json::json!({ "kb_id": kb_id }),
)
.await?;
+ } else if unfinished && reask < MAX_REASK {
+ let delay = std::time::Duration::from_secs(20 * u64::from(reask + 1));
+ tracing::info!(%kb_id, failed, unanswered, reask = reask + 1, delay_secs = delay.as_secs(), "类别词对齐没判完,稍后再问");
+ utopia_store::jobs::enqueue_unless_pending(
+ pool,
+ "align_types",
+ serde_json::json!({ "kb_id": kb_id, "reask": reask + 1 }),
+ delay,
+ )
+ .await?;
+ } else if unfinished {
+ tracing::warn!(%kb_id, failed, unanswered, reask, "类别词对齐问了几轮仍没判完,等下一篇文档或本体改动再问");
}
// 两端的类定了,短语的签名才定:短语对齐排在它后面
utopia_store::jobs::enqueue_unless_queued(
diff --git a/crates/utopia-server/src/type_alignment_tests.rs b/crates/utopia-server/src/type_alignment_tests.rs
index 74b4da12..17c3f63a 100644
--- a/crates/utopia-server/src/type_alignment_tests.rs
+++ b/crates/utopia-server/src/type_alignment_tests.rs
@@ -125,7 +125,7 @@ impl Fx {
}))
}
async fn run(&self) -> anyhow::Result<()> {
- align_types(&self.state, self.kb).await
+ align_types_reasking(&self.state, self.kb, 0).await
}
fn requests(&self) -> Vec<Value> {
self.model.requests.lock().unwrap().clone()
@@ -225,7 +225,7 @@ async fn human_decision_during_disagreement_survives() -> anyhow::Result<()> {
.store(true, std::sync::atomic::Ordering::SeqCst);
let state = f.state.clone();
let kb = f.kb;
- let worker = tokio::spawn(async move { align_types(&state, kb).await });
+ let worker = tokio::spawn(async move { align_types_reasking(&state, kb, 0).await });
tokio::time::timeout(
std::time::Duration::from_secs(10),
f.model.entered.notified(),
@@ -640,3 +640,79 @@ mod review_locks {
result
}
}
+
+/// bench 里「12 篇文档 type_id 全空」的回复:DeepSeek-V3.2 把整段答成按 id 作键的对象,
+/// 第二票还把键包进单元素数组。两票都得读出来,词才绑得上
+#[tokio::test]
+async fn an_id_keyed_reply_still_binds_the_kind_word() -> anyhow::Result<()> {
+ let Some(f) = Fx::new(vec![
+ json!({"0": "organization"}),
+ json!({"b": {"0": ["organization"]}}),
+ ])
+ .await?
+ else {
+ return Ok(());
+ };
+ f.run().await?;
+ let binding = type_bindings::bindings(&f.pool, f.kb).await?.remove(0);
+ let projected: Option<Uuid> = sqlx::query_scalar("SELECT type_id FROM entities WHERE id=$1")
+ .bind(f.entity)
+ .fetch_one(&f.pool)
+ .await?;
+ let count = f.requests().len();
+ let class = f.class;
+ f.cleanup().await?;
+ assert_eq!(count, 2);
+ assert_eq!(binding.status, "bound");
+ assert_eq!(binding.type_id, Some(class));
+ assert_eq!(projected, Some(class));
+ Ok(())
+}
+
+/// 读不出的回复不再是「一项都没答」然后没了:这一轮不下结论,任务自己再排一份
+/// (带 reask),排够 MAX_REASK 次就停,等下一篇文档或本体改动。
+#[tokio::test]
+async fn an_unreadable_reply_is_asked_again_a_bounded_number_of_times() -> anyhow::Result<()> {
+ let Some(f) = Fx::new(vec![
+ json!({"answer": "organization"}),
+ json!({"answer": "organization"}),
+ json!({"answer": "organization"}),
+ json!({"answer": "organization"}),
+ ])
+ .await?
+ else {
+ return Ok(());
+ };
+ f.run().await?;
+ let bindings = type_bindings::bindings(&f.pool, f.kb).await?.len();
+ let reasks: Vec<(String, Option<i64>)> = sqlx::query_as(
+ "SELECT status, (payload->>'reask')::bigint FROM jobs
+ WHERE kind='align_types' AND payload->>'kb_id'=$1 ORDER BY id",
+ )
+ .bind(f.kb.to_string())
+ .fetch_all(&f.pool)
+ .await?;
+ sqlx::query("DELETE FROM jobs WHERE payload->>'kb_id'=$1")
+ .bind(f.kb.to_string())
+ .execute(&f.pool)
+ .await?;
+ // 已经是最后一次自己排的:不再排
+ align_types_reasking(&f.state, f.kb, MAX_REASK).await?;
+ let after: i64 = sqlx::query_scalar(
+ "SELECT count(*) FROM jobs WHERE kind='align_types' AND payload->>'kb_id'=$1",
+ )
+ .bind(f.kb.to_string())
+ .fetch_one(&f.pool)
+ .await?;
+ let count = f.requests().len();
+ f.cleanup().await?;
+ assert_eq!(count, 4, "两轮各问两票");
+ assert_eq!(bindings, 0, "读不出的回复不写任何判定");
+ assert_eq!(
+ reasks,
+ vec![("queued".to_string(), Some(1))],
+ "第一轮之后自己排一份 reask=1"
+ );
+ assert_eq!(after, 0, "排够次数就不再排");
+ Ok(())
+}