World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
The per-chunk ontology list keeps to the prompt budget (#704)
WaylandYangadbbfbbc6288
diff --git a/Cargo.lock b/Cargo.lockindex c3156a85..921b95b8 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -5414,6 +5414,7 @@ dependencies = ["serde","serde_json","tracing",+ "unicode-segmentation","utopia-llm",]diff --git a/Cargo.toml b/Cargo.tomlindex 5afe997b..2db6a439 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -63,6 +63,8 @@ tantivy-jieba = "0.20"text-splitter = { version = "0.32", features = ["tiktoken-rs"] }# 分块预算按 token 数:cl100k 的排名表随 crate 内嵌,离线可用tiktoken-rs = "0.12"+# 句子边界按 Unicode 标准(UAX #29)切,不自己写标点规则+unicode-segmentation = "1.13"pdf-extract = "0.12"calamine = "0.36"pgvector = { version = "0.4", features = ["sqlx"] }diff --git a/crates/utopia-extract/Cargo.toml b/crates/utopia-extract/Cargo.tomlindex ee407ae0..5a0840ce 100644--- a/crates/utopia-extract/Cargo.toml+++ b/crates/utopia-extract/Cargo.toml@@ -11,3 +11,4 @@ anyhow.workspace = truechrono.workspace = trueutopia-llm.workspace = truetracing.workspace = true+unicode-segmentation.workspace = truediff --git a/crates/utopia-extract/src/lib.rs b/crates/utopia-extract/src/lib.rsindex 52b01bcf..428bb07c 100644--- a/crates/utopia-extract/src/lib.rs+++ b/crates/utopia-extract/src/lib.rs@@ -470,6 +470,13 @@ fn opening_block(opening: Option<&str>) -> String {)}+/// 一段描述的第一句(句子边界按 UAX #29)。按块检索出的清单只带这一句:schema.org 的+/// 描述后半截多是用法说明与示例,一块铺上百行时它们占了清单的八成+pub fn first_sentence(text: &str) -> &str {+ use unicode_segmentation::UnicodeSegmentation;+ text.trim().unicode_sentences().next().map_or("", str::trim)+}+/// 清单里给关系带的标记:事件 `[event]`、恒常 `[eternal]`;状态不标。/// 认不出的值当状态——数据库的 CHECK 只放这三个进来,这里不再报错fn temporal_mark(temporal: &str) -> Option<&'static str> {@@ -1822,6 +1829,22 @@ mod tests {assert_eq!(normalize_attr_value("number", &json!("about ten")), None);}+ #[test]+ fn a_description_is_cut_at_its_first_sentence() {+ assert_eq!(+ first_sentence(+ " The date on which the CreativeWork was created. See also dateModified.\n\nExample: 2020-01-01."+ ),+ "The date on which the CreativeWork was created."+ );+ assert_eq!(+ first_sentence("一个有名有姓的人。可以是虚构的。"),+ "一个有名有姓的人。"+ );+ assert_eq!(first_sentence("A person"), "A person");+ assert_eq!(first_sentence(" "), "");+ }+#[test]fn parse_time_precisions() {assert_eq!(parse_time("2024").unwrap().1, "year");diff --git a/crates/utopia-server/src/extraction.rs b/crates/utopia-server/src/extraction.rsindex e54a12f7..98827774 100644--- a/crates/utopia-server/src/extraction.rs+++ b/crates/utopia-server/src/extraction.rs@@ -1097,9 +1097,11 @@ async fn run(state: &AppState, document_id: Uuid, proposer: Proposer) -> anyhow:// 提示词大是慢,没有类可选是抽不出东西let lists = if retrieve_per_chunk {match ctx {- Some(v) => chunk_lists(state, doc.kb_id, v, &etypes, &rtypes, &seed_classes)- .await- .unwrap_or(None),+ Some(v) => {+ chunk_lists(state, doc.kb_id, v, &etypes, &rtypes, &seed_classes, budget)+ .await+ .unwrap_or(None)+ }None => None,}} else {@@ -3037,6 +3039,15 @@ fn build_lists(rels: Option<&HashSet<Uuid>>,) -> PromptLists {let picked_class = |id: &Uuid| classes.is_none_or(|s| s.contains(id));+ // 按块检索(有选择集)时描述只带第一句:检索那条路只有大的导入本体才走,一块铺上百行,+ // 描述占清单的八成(#701)。全铺的是装得下预算的小本体,描述原样+ let describe = |d: &str| -> String {+ if classes.is_some() {+ utopia_extract::first_sentence(d).to_string()+ } else {+ d.to_string()+ }+ };// 边上能带的属性:关系.qualifiers → 属性行(0037)。这里只排版,写入侧另有一份同样的查法let rtype_by_id: HashMap<Uuid, &utopia_core::models::RelationType> =rtypes.iter().map(|r| (r.id, r)).collect();@@ -3064,7 +3075,7 @@ fn build_lists(let types = etypes.iter().filter(|t| picked_class(&t.id))- .map(|t| (t.key.clone(), t.label.clone(), t.description.clone()))+ .map(|t| (t.key.clone(), t.label.clone(), describe(&t.description))).collect();// 一侧的类一个都没铺出去就写 `*`:签名是导向,指向看不见的类只会误导@@ -3092,7 +3103,7 @@ fn build_lists(utopia_extract::PromptRelation {key: r.key.clone(),label: r.label.clone(),- description: r.description.clone(),+ description: describe(&r.description),signature,temporal: r.temporal.clone(),// `amount: number $`——模型要按这个 key 写,单位提醒它别换算@@ -3112,7 +3123,8 @@ fn build_lists(Some(u) if !u.is_empty() => format!("{dt}, {u}"),_ => dt.to_string(),};- let d = r.description.trim();+ let d = describe(&r.description);+ let d = d.trim();Some(if d.is_empty() {format!("- {class_key}.{} ({spec})", r.key)} else {@@ -3151,6 +3163,7 @@ async fn chunk_lists(etypes: &[utopia_core::models::EntityType],rtypes: &[utopia_core::models::RelationType],seed_classes: &HashSet<Uuid>,+ budget: usize,) -> anyhow::Result<Option<PromptLists>> {let mut classes: HashSet<Uuid> = seed_classes.clone();classes.extend(@@ -3186,10 +3199,9 @@ async fn chunk_lists(let picked: Vec<Uuid> = classes.iter().copied().collect();classes.extend(utopia_store::ontology::ancestors_of(&state.pool, &picked).await?);}- let mut rels: HashSet<Uuid> = HashSet::new();// 关系与属性分开检索:两段在提示词里是分开的,混在一起取会让其中一段- // 被另一段挤空- rels.extend(+ // 被另一段挤空。检索回来是按距离排好的,排队时交替取,两段一起往下让+ let nearest = interleave(utopia_store::ontology::nearest_relation_type_ids(&state.pool,kb_id,@@ -3198,8 +3210,6 @@ async fn chunk_lists(Some("relation"),).await?,- );- rels.extend(utopia_store::ontology::nearest_relation_type_ids(&state.pool,kb_id,@@ -3209,6 +3219,7 @@ async fn chunk_lists().await?,);+ let rels: HashSet<Uuid> = nearest.iter().copied().collect();// **一个关系被铺出去,它签名点名的类就得跟着铺。**//// 类与关系是各自独立检索的,而签名依赖两者的交集——`sig_of` 只认铺出去的类,@@ -3225,12 +3236,15 @@ async fn chunk_lists(// 顺带还对:这些类正是模型马上要用来判类型的那些,`employee` 在场就说明// 这一块讲的是雇佣,`organization`/`person` 本来就该在候选里——// 按字面相似度捞不到它们,但**本体的结构知道**。- let sig_classes: HashSet<Uuid> = rtypes- .iter()- .filter(|r| rels.contains(&r.id))- .flat_map(|r| r.domains.iter().chain(r.ranges.iter()).copied())- .collect();- classes.extend(sig_classes);+ let signature_classes = |kept: &HashSet<Uuid>| -> HashSet<Uuid> {+ rtypes+ .iter()+ .filter(|r| kept.contains(&r.id))+ .flat_map(|r| r.domains.iter().chain(r.ranges.iter()).copied())+ .collect()+ };+ let mut domain_pool = classes.clone();+ domain_pool.extend(signature_classes(&rels));// **类进来了,就把本体声明在它们身上的关系也铺出去。**//@@ -3250,12 +3264,13 @@ async fn chunk_lists(// 涨到 21.4k——为了一个谓词付两倍的钱。它们的 domain 侧本来就在清单里// (地板正是这么选出来的),range 侧退化成 `*` 可以接受:这道地板要办的事// 是「让模型看见这个说法存在」,不是把签名补全。- let domain_ids: Vec<Uuid> = classes.iter().copied().collect();+ let domain_ids: Vec<Uuid> = domain_pool.iter().copied().collect();+ let mut on_domains = Vec::new();for (limit, kind) in [(PER_CHUNK_DOMAIN_RELATIONS, "relation"),(PER_CHUNK_DOMAIN_ATTRIBUTES, "attribute"),] {- rels.extend(+ on_domains.push(utopia_store::ontology::nearest_relation_type_ids_in_domains(&state.pool,kb_id,@@ -3267,17 +3282,67 @@ async fn chunk_lists(.await?,);}+ let domain_attributes = on_domains.pop().unwrap_or_default();+ let domain_relations = on_domains.pop().unwrap_or_default();++ // 排队:按相似度检索到的在前,地板补进来的在后+ let mut seen = rels.clone();+ let mut ranked = nearest;+ ranked.extend(+ interleave(domain_relations, domain_attributes)+ .into_iter()+ .filter(|id| seen.insert(*id)),+ );// 一个候选都没检索到 = 索引还没建好,退回全量而不是给一份空清单- if classes.len() <= seed_classes.len() && rels.is_empty() {+ if classes.len() <= seed_classes.len() && ranked.is_empty() {return Ok(None);}- Ok(Some(build_lists(- etypes,- rtypes,- Some(&classes),- Some(&rels),- )))++ // **这一块的清单也守那个预算**(#701)。预算原本只判「全铺装不装得下」,检索出来的+ // 清单没有上限:三道地板叠上去,schema.org 一块铺到 5.6 万字符,是预算的 2.3 倍,+ // 提示词两万 token 里正文不到 2%。+ //+ // 按排队顺序取前 k 个,每个带着它的结构(签名点名的类;祖先在类清单里已经有了),量+ // 实际排出来的那段字,取装得下的最大 k。多一个候选只会多几行,长度随 k 单调,二分就行。+ // 地板补进来的关系不拉签名类,理由见上+ let lists_for = |k: usize| {+ let kept: HashSet<Uuid> = ranked[..k].iter().copied().collect();+ let near_kept: HashSet<Uuid> = kept.intersection(&rels).copied().collect();+ let mut picked = classes.clone();+ picked.extend(signature_classes(&near_kept));+ build_lists(etypes, rtypes, Some(&picked), Some(&kept))+ };+ let (mut lo, mut hi) = (0usize, ranked.len());+ while lo < hi {+ let mid = (lo + hi).div_ceil(2);+ if lists_for(mid).chars() <= budget {+ lo = mid;+ } else {+ hi = mid - 1;+ }+ }+ if lo < ranked.len() {+ tracing::debug!(+ kept = lo,+ retrieved = ranked.len(),+ budget,+ "按块清单按预算截断"+ );+ }+ Ok(Some(lists_for(lo)))+}++/// 两串按距离排好的 id 交替并成一串:a0 b0 a1 b1 …,短的那串用完就接着排长的+fn interleave(a: Vec<Uuid>, b: Vec<Uuid>) -> Vec<Uuid> {+ let mut out = Vec::with_capacity(a.len() + b.len());+ let (mut a, mut b) = (a.into_iter(), b.into_iter());+ loop {+ match (a.next(), b.next()) {+ (None, None) => return out,+ (x, y) => out.extend(x.into_iter().chain(y)),+ }+ }}/// 一条值该记什么单位(#600「单位是读出来的,不是猜的」,实体上的属性与边上的属性同一条规矩)。@@ -3531,6 +3596,17 @@ mod name_tests {#[cfg(test)]mod tests {+ #[test]+ fn two_ranked_lists_take_turns_and_the_longer_one_finishes() {+ let ids: Vec<Uuid> = (0..5).map(|_| Uuid::now_v7()).collect();+ let (a, b) = (vec![ids[0], ids[1], ids[2]], vec![ids[3]]);+ assert_eq!(+ super::interleave(a, b),+ vec![ids[0], ids[3], ids[1], ids[2]]+ );+ assert!(super::interleave(Vec::new(), Vec::new()).is_empty());+ }+#[test]fn a_name_declared_for_another_entity_is_not_an_alias() {let (probe, project) = (Uuid::now_v7(), Uuid::now_v7());diff --git a/docs/decisions/0006-ontology-scale-and-the-prompt.md b/docs/decisions/0006-ontology-scale-and-the-prompt.mdindex e57ac1df..20a6f86e 100644--- a/docs/decisions/0006-ontology-scale-and-the-prompt.md+++ b/docs/decisions/0006-ontology-scale-and-the-prompt.md@@ -2,7 +2,8 @@- **Status**: Built · the character budget (`deployment_settings.ontology_prompt_budget`,24,000) and per-chunk retrieval are live, values unchanged; the "built-in classes always- present" floor is replaced by ancestor completion; answer keys are still hand-filled.+ present" floor is replaced by ancestor completion; the per-chunk list keeps to the same budget+ and carries first sentences (#701); answer keys are still hand-filled.- **Written**: 2026-08-29 · condensed into English 2026-09-03- **Related**: [0008](0008-ontology-packs-as-cold-start.md) packs are now the startingontology; [0012](0012-the-ontology-is-a-contract-not-a-suggestion.md) measured the bias@@ -64,6 +65,17 @@ classes), hence no hit rate.ontology extraction never saw; 25 vs 18 was run variance. The bench now reports`ontology_at_extraction` and `ontology_at_resolution` separately and has`--ontology-first`.+- 2026-09-14 (#701): **the per-chunk list keeps to the budget.** The budget only decided+ whether the whole ontology fits; the list retrieval produced had no limit, and the floors+ added since (ancestors, signature classes, relations declared on the retrieved classes) laid+ out 56,155 characters for one schema.org chunk, 2.3 times the budget, 85% of a 19,735-token+ prompt in which the text was under 2%. Candidates now queue by distance (retrieved first,+ floor additions after, relations and attributes taking turns), each bringing its signature+ classes, and the list takes the longest prefix whose laid-out text fits. A retrieved list+ carries each description's first sentence (UAX #29); descriptions were 84% of it. A full+ inline list is a small ontology and keeps its descriptions. Recall bench 48/52 against 47/52+ before; prompt per extraction call on the same four filings 13.5k → 8.1k tokens, measured+ together with the sentence-numbered reply.- 2026-09-02: per-chunk retrieval sends one vector query per chunk with concurrency up to`worker_concurrency` (cap 256 since #133) against a pool of 32; migration `0011` recordsit.diff --git a/docs/decisions/README.md b/docs/decisions/README.mdindex cdac2d9d..5f372dc8 100644--- a/docs/decisions/README.md+++ b/docs/decisions/README.md@@ -29,7 +29,7 @@ The test for writing one: if someone (including us) looks at a piece of code in| 0003 | [The ontology grows out of the corpus](0003-ontology-growth-loop.md) | Built and running, default on · starting point rewritten by 0010 and the retired seeds · dismissal redone per 0007 · the "new phrasings" reminder pending || 0004 | [Language follows the reader of each text](0004-language-and-localization.md) | Built · UI strings, coded server errors, ontology description language and the locale of generated text · the browser language is not guessed yet || 0005 | [The alert center](0005-alert-center.md) | Built · five alert kinds live, search and paging in the panel · `document.no_text_layer` still unwired |-| 0006 | [Ontology scale and the extraction prompt](0006-ontology-scale-and-the-prompt.md) | Built · the character budget (24,000) and per-chunk retrieval live, values untested · answer keys still hand-filled |+| 0006 | [Ontology scale and the extraction prompt](0006-ontology-scale-and-the-prompt.md) | Built · the character budget (24,000) and per-chunk retrieval live, values untested · the per-chunk list keeps to the same budget with first-sentence descriptions (#701) · answer keys still hand-filled || 0007 | [Counting decides what becomes a relation](0007-who-decides-what-becomes-a-relation.md) | Built · adoption decided by counting (`MIN_DOCS = 2`, `MIN_SIGNALS = 3`), proposals persist (#112) · narrative verbs and `_by` folding still open || 0008 | [Ontology packs as the cold start](0008-ontology-packs-as-cold-start.md) | Built · five packs embedded, multi-select at creation · no pack by default and none at registration (#580, measured) · three open questions stay open; Chinese labels got worse || 0009 | [An undecided type stays empty](0009-no-type-is-a-type.md) | Implemented · `type_id` nullable, builtin classes gone · kin classes go to Review (#226), declared `disjointWith` keeps them apart (0016 B3) · `metric` / `dimension` builtin on demand (#231) · `metric` / `dimension` to retire under 0036 |