Skip to content

World's first open-source enterprise world model.

The definitions of the candidate classes are written once per batch of kind words (#1004)

WaylandYang2de0dcf3796e
diff --git a/crates/utopia-extract/src/align.rs b/crates/utopia-extract/src/align.rs
index a9e4e33e..fec25043 100644
--- a/crates/utopia-extract/src/align.rs
+++ b/crates/utopia-extract/src/align.rs
@@ -63,9 +63,9 @@ const KIND_WORD_SYSTEM: &str = "\
You bind the words documents use for kinds of things to the classes of a knowledge base's \
ontology. Each numbered item is one kind word as the documents wrote it: the word, the \
spellings seen, some things of this kind by name, the phrases those things take part in, and \
-the candidate classes, each with its key, its label and its definition. For each item, answer \
-with the key of the one candidate class that every thing of this kind is an instance of, or \
-null.\n\
+the keys of its candidate classes. The classes themselves are listed once under \"Classes\", \
+each with its key, its label and its definition. For each item, answer with the key of the one \
+candidate class that every thing of this kind is an instance of, or null.\n\
\n\
Output exactly one JSON object and nothing else, one pair per item:\n\
{\"b\": [[12, \"organization\"], [13, null]]}\n\
@@ -90,16 +90,31 @@ documents: things that \"were founded by\" and \"are based in\" are organization
5. Never invent a key, never answer with a label or a word of your own, and never choose for \
an item a key that is not among its candidates. One pair per item, every item answered.";
-/// 构造两条消息:常量系统消息 + 逐项的用户消息。每项:id、词、拼写、例子、短语、候选。
-/// 空清单写 `(none)`,模型知道那一栏存在但这次没有证据
+/// 构造两条消息:常量系统消息 + 用户消息。类的定义表一批只写一遍,按第一次出现的次序;
+/// 每项:id、词、拼写、例子、短语、候选的键(次序照给的,第二票倒着给就倒着列)。
+/// 空清单写 `(none)`,模型知道那一栏存在但这次没有证据。
+///
+/// 从前每项下面各列一遍候选的定义:一批二十个词、每个十条候选,同一张表写二十遍,
+/// 一个词一百四十多 token 里一百二是它(bench README,2026-09-29)。与
+/// `phrase_align::build_phrase_messages` 同一条理由
pub fn build_kind_word_messages(items: &[KindWordItem<'_>]) -> Vec<ChatMessage> {
+ let mut glossary: Vec<String> = Vec::new();
+ let mut seen = std::collections::HashSet::new();
+ for c in items.iter().flat_map(|i| i.candidates.iter()) {
+ if seen.insert(c.key.trim()) {
+ glossary.push(candidate_line(c));
+ }
+ }
let mut user = String::new();
+ if !glossary.is_empty() {
+ user.push_str(&format!("Classes:\n{}\n\n", glossary.join("\n")));
+ }
for item in items {
let candidates = if item.candidates.is_empty() {
" (none)".to_string()
} else {
- let lines: Vec<String> = item.candidates.iter().map(candidate_line).collect();
- format!("\n{}", lines.join("\n"))
+ let keys: Vec<&str> = item.candidates.iter().map(|c| c.key.trim()).collect();
+ format!(" {}", keys.join(", "))
};
user.push_str(&format!(
"Item {}: \"{}\"\nSpellings: {}\nExamples: {}\nPhrases: {}\nCandidates:{candidates}\n\n",
@@ -629,10 +644,18 @@ mod tests {
assert!(user.contains("Spellings: \"Company\", \"company\", \"公司\"\n"));
assert!(user.contains("Examples: \"Brightway Builders\", \"Harbor Estates\"\n"));
assert!(user.contains("Phrases: \"is based in\", \"was founded by\"\n"));
- assert!(user.contains(
- "- organization: Organization — An organized group of people with a shared purpose"
- ));
- assert!(user.contains("- person: Person — A human being."));
+ assert!(user.starts_with("Classes:\n- "));
+ assert_eq!(
+ user.matches("- organization: Organization — An organized group of people")
+ .count(),
+ 1,
+ "一条定义一批只写一遍"
+ );
+ assert_eq!(user.matches("- person: Person — A human being.").count(), 1);
+ assert!(
+ user.find("- structure: ").unwrap() < user.find("Item 12").unwrap(),
+ "定义表在各项之前"
+ );
assert!(user.contains("Item 13: \"participant\"\n"));
assert!(user.contains("Spellings: \"participants\"\nExamples: (none)\nPhrases: (none)\n"));
@@ -680,7 +703,8 @@ mod tests {
candidates: vec![],
});
let user = &build_kind_word_messages(&items)[1].content;
- assert!(user.contains("Candidates:\n- organization\n- place — A location.\n"));
+ assert!(user.starts_with("Classes:\n- organization\n- place — A location.\n\n"));
+ assert!(user.contains("Candidates: organization, place\n"));
assert!(user.contains("Item 15: \"council\"\n"));
assert!(user.ends_with("Candidates: (none)"));
}
diff --git a/scripts/bench/README.md b/scripts/bench/README.md
index 8d233182..1f971338 100644
--- a/scripts/bench/README.md
+++ b/scripts/bench/README.md
@@ -915,3 +915,26 @@ bound / none / undecided。0044 的门槛:裁判精度不低于完整原型的
| 另外 3 份(坏在结尾) | | 不变 | 不变 |
整体合法的 365 份,新旧解析逐条相同。
+
+### 第八次真跑:类的定义表一批只写一遍(2026-09-29,dev @ 8109661,gemini-3.5-flash)
+
+类别词对齐每篇 4314 token。一批二十个词,每个词下面各列一遍候选类的定义:一个词约 145 token 的输入里
+约 120 是它。改成定义表一批写一遍、各项只列候选的键(与短语对齐同一个办法),判断的规则不动。
+
+在第七次那个库上(20 篇,176 个类别词,实体不动)把类别词的绑定清掉重判,新旧各两遍。正确率是库里的实体
+按名字对上金标实体之后,类别词绑到的类与金标类型是否一致(216 个对得上、类型不含糊的实体):
+
+| | 从前 | 从前 再一遍 | 定义表写一遍 | 再一遍 |
+|---|---|---|---|---|
+| 调用 | 18 | 18 | 18 | 18 |
+| 输入 token | 61382 | 61389 | 29142 | 29138 |
+| 输出 token | 19977 | 16361 | 13748 | 14571 |
+| 绑定 bound / none / undecided | 163 / 2 / 11 | 162 / 3 / 11 | 157 / 1 / 18 | 166 / 2 / 8 |
+| 实体 对 / 错 / 没绑上 | 210 / 4 / 2 | 210 / 5 / 1 | 207 / 6 / 3 | 209 / 6 / 1 |
+| 绑上的里面对的 | 98.1% | 97.7% | 97.2% | 97.2% |
+
+- 输入少一半多(6.1 万 → 2.9 万),合计 8.0 万 → 4.3 万。
+- 四遍里判得不一样的 25 个词,没有一个是两遍绑到了不同的类:全是绑上与拿不定之间来回。同一个构建两遍
+ 相同的 166 与 160 个,新旧之间 157 到 164 个,分不出来。
+- 正确率差一个实体上下(错 4、5 对 6、6),在这个量上也分不出来;本体只有六个类,候选多、定义长的
+ 本体省得更多,也更该在那样的库上再量一次。