Skip to content

World's first open-source enterprise world model.

The per-chunk ontology list keeps to the prompt budget (#704)

WaylandYangadbbfbbc6288
diff --git a/Cargo.lock b/Cargo.lock
index c3156a85..921b95b8 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5414,6 +5414,7 @@ dependencies = [
"serde",
"serde_json",
"tracing",
+ "unicode-segmentation",
"utopia-llm",
]
diff --git a/Cargo.toml b/Cargo.toml
index 5afe997b..2db6a439 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -63,6 +63,8 @@ tantivy-jieba = "0.20"
text-splitter = { version = "0.32", features = ["tiktoken-rs"] }
# 分块预算按 token 数:cl100k 的排名表随 crate 内嵌,离线可用
tiktoken-rs = "0.12"
+# 句子边界按 Unicode 标准(UAX #29)切,不自己写标点规则
+unicode-segmentation = "1.13"
pdf-extract = "0.12"
calamine = "0.36"
pgvector = { version = "0.4", features = ["sqlx"] }
diff --git a/crates/utopia-extract/Cargo.toml b/crates/utopia-extract/Cargo.toml
index ee407ae0..5a0840ce 100644
--- a/crates/utopia-extract/Cargo.toml
+++ b/crates/utopia-extract/Cargo.toml
@@ -11,3 +11,4 @@ anyhow.workspace = true
chrono.workspace = true
utopia-llm.workspace = true
tracing.workspace = true
+unicode-segmentation.workspace = true
diff --git a/crates/utopia-extract/src/lib.rs b/crates/utopia-extract/src/lib.rs
index 52b01bcf..428bb07c 100644
--- a/crates/utopia-extract/src/lib.rs
+++ b/crates/utopia-extract/src/lib.rs
@@ -470,6 +470,13 @@ fn opening_block(opening: Option<&str>) -> String {
)
}
+/// 一段描述的第一句(句子边界按 UAX #29)。按块检索出的清单只带这一句:schema.org 的
+/// 描述后半截多是用法说明与示例,一块铺上百行时它们占了清单的八成
+pub fn first_sentence(text: &str) -> &str {
+ use unicode_segmentation::UnicodeSegmentation;
+ text.trim().unicode_sentences().next().map_or("", str::trim)
+}
+
/// 清单里给关系带的标记:事件 `[event]`、恒常 `[eternal]`;状态不标。
/// 认不出的值当状态——数据库的 CHECK 只放这三个进来,这里不再报错
fn temporal_mark(temporal: &str) -> Option<&'static str> {
@@ -1822,6 +1829,22 @@ mod tests {
assert_eq!(normalize_attr_value("number", &json!("about ten")), None);
}
+ #[test]
+ fn a_description_is_cut_at_its_first_sentence() {
+ assert_eq!(
+ first_sentence(
+ " The date on which the CreativeWork was created. See also dateModified.\n\nExample: 2020-01-01."
+ ),
+ "The date on which the CreativeWork was created."
+ );
+ assert_eq!(
+ first_sentence("一个有名有姓的人。可以是虚构的。"),
+ "一个有名有姓的人。"
+ );
+ assert_eq!(first_sentence("A person"), "A person");
+ assert_eq!(first_sentence(" "), "");
+ }
+
#[test]
fn parse_time_precisions() {
assert_eq!(parse_time("2024").unwrap().1, "year");
diff --git a/crates/utopia-server/src/extraction.rs b/crates/utopia-server/src/extraction.rs
index e54a12f7..98827774 100644
--- a/crates/utopia-server/src/extraction.rs
+++ b/crates/utopia-server/src/extraction.rs
@@ -1097,9 +1097,11 @@ async fn run(state: &AppState, document_id: Uuid, proposer: Proposer) -> anyhow:
// 提示词大是慢,没有类可选是抽不出东西
let lists = if retrieve_per_chunk {
match ctx {
- Some(v) => chunk_lists(state, doc.kb_id, v, &etypes, &rtypes, &seed_classes)
- .await
- .unwrap_or(None),
+ Some(v) => {
+ chunk_lists(state, doc.kb_id, v, &etypes, &rtypes, &seed_classes, budget)
+ .await
+ .unwrap_or(None)
+ }
None => None,
}
} else {
@@ -3037,6 +3039,15 @@ fn build_lists(
rels: Option<&HashSet<Uuid>>,
) -> PromptLists {
let picked_class = |id: &Uuid| classes.is_none_or(|s| s.contains(id));
+ // 按块检索(有选择集)时描述只带第一句:检索那条路只有大的导入本体才走,一块铺上百行,
+ // 描述占清单的八成(#701)。全铺的是装得下预算的小本体,描述原样
+ let describe = |d: &str| -> String {
+ if classes.is_some() {
+ utopia_extract::first_sentence(d).to_string()
+ } else {
+ d.to_string()
+ }
+ };
// 边上能带的属性:关系.qualifiers → 属性行(0037)。这里只排版,写入侧另有一份同样的查法
let rtype_by_id: HashMap<Uuid, &utopia_core::models::RelationType> =
rtypes.iter().map(|r| (r.id, r)).collect();
@@ -3064,7 +3075,7 @@ fn build_lists(
let types = etypes
.iter()
.filter(|t| picked_class(&t.id))
- .map(|t| (t.key.clone(), t.label.clone(), t.description.clone()))
+ .map(|t| (t.key.clone(), t.label.clone(), describe(&t.description)))
.collect();
// 一侧的类一个都没铺出去就写 `*`:签名是导向,指向看不见的类只会误导
@@ -3092,7 +3103,7 @@ fn build_lists(
utopia_extract::PromptRelation {
key: r.key.clone(),
label: r.label.clone(),
- description: r.description.clone(),
+ description: describe(&r.description),
signature,
temporal: r.temporal.clone(),
// `amount: number $`——模型要按这个 key 写,单位提醒它别换算
@@ -3112,7 +3123,8 @@ fn build_lists(
Some(u) if !u.is_empty() => format!("{dt}, {u}"),
_ => dt.to_string(),
};
- let d = r.description.trim();
+ let d = describe(&r.description);
+ let d = d.trim();
Some(if d.is_empty() {
format!("- {class_key}.{} ({spec})", r.key)
} else {
@@ -3151,6 +3163,7 @@ async fn chunk_lists(
etypes: &[utopia_core::models::EntityType],
rtypes: &[utopia_core::models::RelationType],
seed_classes: &HashSet<Uuid>,
+ budget: usize,
) -> anyhow::Result<Option<PromptLists>> {
let mut classes: HashSet<Uuid> = seed_classes.clone();
classes.extend(
@@ -3186,10 +3199,9 @@ async fn chunk_lists(
let picked: Vec<Uuid> = classes.iter().copied().collect();
classes.extend(utopia_store::ontology::ancestors_of(&state.pool, &picked).await?);
}
- let mut rels: HashSet<Uuid> = HashSet::new();
// 关系与属性分开检索:两段在提示词里是分开的,混在一起取会让其中一段
- // 被另一段挤空
- rels.extend(
+ // 被另一段挤空。检索回来是按距离排好的,排队时交替取,两段一起往下让
+ let nearest = interleave(
utopia_store::ontology::nearest_relation_type_ids(
&state.pool,
kb_id,
@@ -3198,8 +3210,6 @@ async fn chunk_lists(
Some("relation"),
)
.await?,
- );
- rels.extend(
utopia_store::ontology::nearest_relation_type_ids(
&state.pool,
kb_id,
@@ -3209,6 +3219,7 @@ async fn chunk_lists(
)
.await?,
);
+ let rels: HashSet<Uuid> = nearest.iter().copied().collect();
// **一个关系被铺出去,它签名点名的类就得跟着铺。**
//
// 类与关系是各自独立检索的,而签名依赖两者的交集——`sig_of` 只认铺出去的类,
@@ -3225,12 +3236,15 @@ async fn chunk_lists(
// 顺带还对:这些类正是模型马上要用来判类型的那些,`employee` 在场就说明
// 这一块讲的是雇佣,`organization`/`person` 本来就该在候选里——
// 按字面相似度捞不到它们,但**本体的结构知道**。
- let sig_classes: HashSet<Uuid> = rtypes
- .iter()
- .filter(|r| rels.contains(&r.id))
- .flat_map(|r| r.domains.iter().chain(r.ranges.iter()).copied())
- .collect();
- classes.extend(sig_classes);
+ let signature_classes = |kept: &HashSet<Uuid>| -> HashSet<Uuid> {
+ rtypes
+ .iter()
+ .filter(|r| kept.contains(&r.id))
+ .flat_map(|r| r.domains.iter().chain(r.ranges.iter()).copied())
+ .collect()
+ };
+ let mut domain_pool = classes.clone();
+ domain_pool.extend(signature_classes(&rels));
// **类进来了,就把本体声明在它们身上的关系也铺出去。**
//
@@ -3250,12 +3264,13 @@ async fn chunk_lists(
// 涨到 21.4k——为了一个谓词付两倍的钱。它们的 domain 侧本来就在清单里
// (地板正是这么选出来的),range 侧退化成 `*` 可以接受:这道地板要办的事
// 是「让模型看见这个说法存在」,不是把签名补全。
- let domain_ids: Vec<Uuid> = classes.iter().copied().collect();
+ let domain_ids: Vec<Uuid> = domain_pool.iter().copied().collect();
+ let mut on_domains = Vec::new();
for (limit, kind) in [
(PER_CHUNK_DOMAIN_RELATIONS, "relation"),
(PER_CHUNK_DOMAIN_ATTRIBUTES, "attribute"),
] {
- rels.extend(
+ on_domains.push(
utopia_store::ontology::nearest_relation_type_ids_in_domains(
&state.pool,
kb_id,
@@ -3267,17 +3282,67 @@ async fn chunk_lists(
.await?,
);
}
+ let domain_attributes = on_domains.pop().unwrap_or_default();
+ let domain_relations = on_domains.pop().unwrap_or_default();
+
+ // 排队:按相似度检索到的在前,地板补进来的在后
+ let mut seen = rels.clone();
+ let mut ranked = nearest;
+ ranked.extend(
+ interleave(domain_relations, domain_attributes)
+ .into_iter()
+ .filter(|id| seen.insert(*id)),
+ );
// 一个候选都没检索到 = 索引还没建好,退回全量而不是给一份空清单
- if classes.len() <= seed_classes.len() && rels.is_empty() {
+ if classes.len() <= seed_classes.len() && ranked.is_empty() {
return Ok(None);
}
- Ok(Some(build_lists(
- etypes,
- rtypes,
- Some(&classes),
- Some(&rels),
- )))
+
+ // **这一块的清单也守那个预算**(#701)。预算原本只判「全铺装不装得下」,检索出来的
+ // 清单没有上限:三道地板叠上去,schema.org 一块铺到 5.6 万字符,是预算的 2.3 倍,
+ // 提示词两万 token 里正文不到 2%。
+ //
+ // 按排队顺序取前 k 个,每个带着它的结构(签名点名的类;祖先在类清单里已经有了),量
+ // 实际排出来的那段字,取装得下的最大 k。多一个候选只会多几行,长度随 k 单调,二分就行。
+ // 地板补进来的关系不拉签名类,理由见上
+ let lists_for = |k: usize| {
+ let kept: HashSet<Uuid> = ranked[..k].iter().copied().collect();
+ let near_kept: HashSet<Uuid> = kept.intersection(&rels).copied().collect();
+ let mut picked = classes.clone();
+ picked.extend(signature_classes(&near_kept));
+ build_lists(etypes, rtypes, Some(&picked), Some(&kept))
+ };
+ let (mut lo, mut hi) = (0usize, ranked.len());
+ while lo < hi {
+ let mid = (lo + hi).div_ceil(2);
+ if lists_for(mid).chars() <= budget {
+ lo = mid;
+ } else {
+ hi = mid - 1;
+ }
+ }
+ if lo < ranked.len() {
+ tracing::debug!(
+ kept = lo,
+ retrieved = ranked.len(),
+ budget,
+ "按块清单按预算截断"
+ );
+ }
+ Ok(Some(lists_for(lo)))
+}
+
+/// 两串按距离排好的 id 交替并成一串:a0 b0 a1 b1 …,短的那串用完就接着排长的
+fn interleave(a: Vec<Uuid>, b: Vec<Uuid>) -> Vec<Uuid> {
+ let mut out = Vec::with_capacity(a.len() + b.len());
+ let (mut a, mut b) = (a.into_iter(), b.into_iter());
+ loop {
+ match (a.next(), b.next()) {
+ (None, None) => return out,
+ (x, y) => out.extend(x.into_iter().chain(y)),
+ }
+ }
}
/// 一条值该记什么单位(#600「单位是读出来的,不是猜的」,实体上的属性与边上的属性同一条规矩)。
@@ -3531,6 +3596,17 @@ mod name_tests {
#[cfg(test)]
mod tests {
+ #[test]
+ fn two_ranked_lists_take_turns_and_the_longer_one_finishes() {
+ let ids: Vec<Uuid> = (0..5).map(|_| Uuid::now_v7()).collect();
+ let (a, b) = (vec![ids[0], ids[1], ids[2]], vec![ids[3]]);
+ assert_eq!(
+ super::interleave(a, b),
+ vec![ids[0], ids[3], ids[1], ids[2]]
+ );
+ assert!(super::interleave(Vec::new(), Vec::new()).is_empty());
+ }
+
#[test]
fn a_name_declared_for_another_entity_is_not_an_alias() {
let (probe, project) = (Uuid::now_v7(), Uuid::now_v7());
diff --git a/docs/decisions/0006-ontology-scale-and-the-prompt.md b/docs/decisions/0006-ontology-scale-and-the-prompt.md
index e57ac1df..20a6f86e 100644
--- a/docs/decisions/0006-ontology-scale-and-the-prompt.md
+++ b/docs/decisions/0006-ontology-scale-and-the-prompt.md
@@ -2,7 +2,8 @@
- **Status**: Built · the character budget (`deployment_settings.ontology_prompt_budget`,
24,000) and per-chunk retrieval are live, values unchanged; the "built-in classes always
- present" floor is replaced by ancestor completion; answer keys are still hand-filled.
+ present" floor is replaced by ancestor completion; the per-chunk list keeps to the same budget
+ and carries first sentences (#701); answer keys are still hand-filled.
- **Written**: 2026-08-29 · condensed into English 2026-09-03
- **Related**: [0008](0008-ontology-packs-as-cold-start.md) packs are now the starting
ontology; [0012](0012-the-ontology-is-a-contract-not-a-suggestion.md) measured the bias
@@ -64,6 +65,17 @@ classes), hence no hit rate.
ontology extraction never saw; 25 vs 18 was run variance. The bench now reports
`ontology_at_extraction` and `ontology_at_resolution` separately and has
`--ontology-first`.
+- 2026-09-14 (#701): **the per-chunk list keeps to the budget.** The budget only decided
+ whether the whole ontology fits; the list retrieval produced had no limit, and the floors
+ added since (ancestors, signature classes, relations declared on the retrieved classes) laid
+ out 56,155 characters for one schema.org chunk, 2.3 times the budget, 85% of a 19,735-token
+ prompt in which the text was under 2%. Candidates now queue by distance (retrieved first,
+ floor additions after, relations and attributes taking turns), each bringing its signature
+ classes, and the list takes the longest prefix whose laid-out text fits. A retrieved list
+ carries each description's first sentence (UAX #29); descriptions were 84% of it. A full
+ inline list is a small ontology and keeps its descriptions. Recall bench 48/52 against 47/52
+ before; prompt per extraction call on the same four filings 13.5k → 8.1k tokens, measured
+ together with the sentence-numbered reply.
- 2026-09-02: per-chunk retrieval sends one vector query per chunk with concurrency up to
`worker_concurrency` (cap 256 since #133) against a pool of 32; migration `0011` records
it.
diff --git a/docs/decisions/README.md b/docs/decisions/README.md
index cdac2d9d..5f372dc8 100644
--- a/docs/decisions/README.md
+++ b/docs/decisions/README.md
@@ -29,7 +29,7 @@ The test for writing one: if someone (including us) looks at a piece of code in
| 0003 | [The ontology grows out of the corpus](0003-ontology-growth-loop.md) | Built and running, default on · starting point rewritten by 0010 and the retired seeds · dismissal redone per 0007 · the "new phrasings" reminder pending |
| 0004 | [Language follows the reader of each text](0004-language-and-localization.md) | Built · UI strings, coded server errors, ontology description language and the locale of generated text · the browser language is not guessed yet |
| 0005 | [The alert center](0005-alert-center.md) | Built · five alert kinds live, search and paging in the panel · `document.no_text_layer` still unwired |
-| 0006 | [Ontology scale and the extraction prompt](0006-ontology-scale-and-the-prompt.md) | Built · the character budget (24,000) and per-chunk retrieval live, values untested · answer keys still hand-filled |
+| 0006 | [Ontology scale and the extraction prompt](0006-ontology-scale-and-the-prompt.md) | Built · the character budget (24,000) and per-chunk retrieval live, values untested · the per-chunk list keeps to the same budget with first-sentence descriptions (#701) · answer keys still hand-filled |
| 0007 | [Counting decides what becomes a relation](0007-who-decides-what-becomes-a-relation.md) | Built · adoption decided by counting (`MIN_DOCS = 2`, `MIN_SIGNALS = 3`), proposals persist (#112) · narrative verbs and `_by` folding still open |
| 0008 | [Ontology packs as the cold start](0008-ontology-packs-as-cold-start.md) | Built · five packs embedded, multi-select at creation · no pack by default and none at registration (#580, measured) · three open questions stay open; Chinese labels got worse |
| 0009 | [An undecided type stays empty](0009-no-type-is-a-type.md) | Implemented · `type_id` nullable, builtin classes gone · kin classes go to Review (#226), declared `disjointWith` keeps them apart (0016 B3) · `metric` / `dimension` builtin on demand (#231) · `metric` / `dimension` to retire under 0036 |