World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
The prompt gets the slice of the ontology this chunk needs (#85)
WaylandYanga0b0ca172085
diff --git a/crates/utopia-server/src/extraction.rs b/crates/utopia-server/src/extraction.rsindex 872eef78..ee4f318a 100644--- a/crates/utopia-server/src/extraction.rs+++ b/crates/utopia-server/src/extraction.rs@@ -107,10 +107,6 @@ async fn run(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {let etypes = utopia_store::graph::entity_types(&state.pool, doc.kb_id).await?;let rtypes = utopia_store::graph::relation_types(&state.pool, doc.kb_id).await?;- let type_pairs: Vec<(String, String, String)> = etypes- .iter()- .map(|t| (t.key.clone(), t.label.clone(), t.description.clone()))- .collect();// 关系与属性分道:属性走字面值通道,不进关系清单。//// related_to 刻意不列给模型:它在本体里是代码层的兜底,但摆进提示词就成了@@ -120,40 +116,10 @@ async fn run(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {// 兜底改由下面的代码执行,原词落进 fact_evidence.proposed_predicate 留待消解。let type_key_by_id: HashMap<Uuid, &str> =etypes.iter().map(|t| (t.id, t.key.as_str())).collect();- // 类型签名:`person|organization → vendor`,一侧为空写 `*`。- // **两侧都为空就不给签名**——那不是"没填",是"不限",硬写一个 `* → *`- // 只会给每个文本块的提示词加一行噪音- let sig_of = |ids: &[Uuid]| -> String {- if ids.is_empty() {- return "*".into();- }- let mut keys: Vec<&str> = ids- .iter()- .filter_map(|id| type_key_by_id.get(id).copied())- .collect();- keys.sort_unstable();- if keys.is_empty() {- "*".into()- } else {- keys.join("|")- }- };- let rel_pairs: Vec<utopia_extract::PromptRelation> = rtypes+ let attr_meta: HashMap<&str, &utopia_core::models::RelationType> = rtypes.iter()- .filter(|r| r.kind != "attribute" && r.key != FALLBACK_RELATION_KEY)- .map(|r| {- let signature = if r.domains.is_empty() && r.ranges.is_empty() {- String::new()- } else {- format!("{} → {}", sig_of(&r.domains), sig_of(&r.ranges))- };- utopia_extract::PromptRelation {- key: r.key.clone(),- label: r.label.clone(),- description: r.description.clone(),- signature,- }- })+ .filter(|r| r.kind == "attribute")+ .map(|r| (r.key.as_str(), r)).collect();let type_ids: HashMap<&str, Uuid> = etypes.iter().map(|t| (t.key.as_str(), t.id)).collect();let rel_ids: HashMap<&str, Uuid> = rtypes.iter().map(|r| (r.key.as_str(), r.id)).collect();@@ -167,38 +133,33 @@ async fn run(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {)}).collect();- // 属性元数据(按 key 查)与提示词清单("person.salary (number, CNY): 月薪")。- // 没定义属性时清单为空,提示词一字不变let type_parents: HashMap<Uuid, &[Uuid]> = etypes.iter().map(|t| (t.id, t.parents.as_slice())).collect();- let attr_meta: HashMap<&str, &utopia_core::models::RelationType> = rtypes- .iter()- .filter(|r| r.kind == "attribute")- .map(|r| (r.key.as_str(), r))- .collect();- // 一个属性挂在多个类下时,**每个类各排一行**:模型读到的是- // "store.opens_at" 而不是一个要它自己去分配的类清单- let attr_lines: Vec<String> = rtypes- .iter()- .filter(|r| r.kind == "attribute")- .flat_map(|r| r.domains.iter().map(move |d| (r, d)))- .filter_map(|(r, domain_id)| {- let class_key = type_key_by_id.get(domain_id)?;- let dt = r.datatype.as_deref().unwrap_or("text");- let spec = match &r.unit {- Some(u) if !u.is_empty() => format!("{dt}, {u}"),- _ => dt.to_string(),- };- let d = r.description.trim();- Some(if d.is_empty() {- format!("- {class_key}.{} ({spec})", r.key)- } else {- format!("- {class_key}.{} ({spec}): {d}", r.key)- })- })- .collect();++ // **本体全铺还是按分块检索。**+ //+ // 全铺是今天的行为,小本体下它对且便宜:40 个类约 2k 字符,检索反而是多余的+ // 往返。大本体下它是灾难——schema.org 实测每个分块 108k tokens,而同一份语料+ // 只给种子类时抽到 25 个实体、给全量时只剩 18 个。**多给的那 959 个类+ // 吃掉了 7 个实体。**+ //+ // 所以按预算切换:装得下就全铺,装不下就每块检索。判据量的是**实际要排的+ // 那段字**(build_lists 自己数),不是另写一个估算公式——公式会跟排版分叉。+ let full = build_lists(&etypes, &rtypes, None, None);+ let budget = utopia_store::access::ontology_prompt_budget(&state.pool).await?;+ let retrieve_per_chunk = full.chars() > budget;+ if retrieve_per_chunk {+ tracing::info!(+ %document_id, chars = full.chars(), budget,+ classes = etypes.len(),+ "本体超出提示词预算,改为按分块检索候选"+ );+ }+ // 内置类恒在:检索漏掉的分块仍要有地方落脚,否则模型无类可选+ let seed_classes: HashSet<Uuid> = etypes.iter().filter(|t| t.builtin).map(|t| t.id).collect();+// 属性 domain 允许子类:主语类型沿 parent 链上溯命中 domain 即可// 沿 subClassOf 上溯。**广度优先 + 访问集**,不是单链循环:// 一个类可以有多个父(FOAF 的 Person 同时是 Agent 与 SpatialThing),@@ -252,14 +213,29 @@ async fn run(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {return Ok(());}let ctx: Option<&[f32]> = chunk.embedding.as_ref().map(|v| v.as_slice());+ // 本体装得下就用全量那份;装不下就拿**这一块自己的向量**检索候选。+ // 向量是现成的——实体消解本来就在用它(上面那个 ctx),检索一次+ // 嵌入都不用加。检索不出来(没配嵌入模型、或这块没向量)就退回全量:+ // 提示词大是慢,没有类可选是抽不出东西+ let lists = if retrieve_per_chunk {+ match ctx {+ Some(v) => chunk_lists(state, doc.kb_id, v, &etypes, &rtypes, &seed_classes)+ .await+ .unwrap_or(None),+ None => None,+ }+ } else {+ None+ };+ let lists = lists.as_ref().unwrap_or(&full);let known: Vec<(String, String)> = doc_entities.iter().map(|(_, k, n)| (k.clone(), n.clone())).collect();let messages = utopia_extract::build_messages(- &type_pairs,- &rel_pairs,- &attr_lines,+ &lists.types,+ &lists.relations,+ &lists.attributes,doc_time.as_deref(),&doc.filename,&known,@@ -818,6 +794,183 @@ fn looks_literal(s: &str) -> bool {utopia_extract::parse_time(s).is_some()}+/// 提示词里那三段清单:类、关系、属性。+///+/// **抽出来是为了让"全给"和"按分块检索"共用同一段排版逻辑。**两条路各排一份+/// 的话迟早分叉,而分叉在这里的后果是提示词说的与代码认的不是一回事。+struct PromptLists {+ types: Vec<(String, String, String)>,+ relations: Vec<utopia_extract::PromptRelation>,+ attributes: Vec<String>,+}++impl PromptLists {+ /// 这三段铺进提示词有多长。budget 判据用它——**量的是实际要排的那段字**,+ /// 不是另写一个估算公式(公式会跟排版分叉)。+ fn chars(&self) -> usize {+ self.types+ .iter()+ .map(|(k, l, d)| k.len() + l.len() + d.len() + 6)+ .sum::<usize>()+ + self+ .relations+ .iter()+ .map(|r| r.key.len() + r.label.len() + r.description.len() + r.signature.len() + 8)+ .sum::<usize>()+ + self.attributes.iter().map(|a| a.len() + 1).sum::<usize>()+ }+}++/// 从一个**选择集**排出三段清单。`None` = 全给(本体小于预算时的老路)。+///+/// 三处细节都是选择带来的,全给时它们不会触发:+///+/// 1. **签名只能提到选中的类**。`works_at (person → organization)` 里那两个 key+/// 必须是模型看得见的——写一个没铺出去的类名,等于教它输出一个不存在的类型。+/// 整侧都没选中就退回 `*`。+/// 2. **属性跟着 domain 走**。属性行是 `class.attr`,它的类没铺出去这行就没意义。+/// 这也顺带解决了属性段(占提示词 28%)的裁剪,不用单独处理。+/// 3. **内置类恒在**。检索漏掉的分块仍然要有地方落脚,否则模型无类可选。+fn build_lists(+ etypes: &[utopia_core::models::EntityType],+ rtypes: &[utopia_core::models::RelationType],+ classes: Option<&HashSet<Uuid>>,+ rels: Option<&HashSet<Uuid>>,+) -> PromptLists {+ let picked_class = |id: &Uuid| classes.is_none_or(|s| s.contains(id));+ let picked_rel = |id: &Uuid| rels.is_none_or(|s| s.contains(id));+ let key_of: HashMap<Uuid, &str> = etypes+ .iter()+ .filter(|t| picked_class(&t.id))+ .map(|t| (t.id, t.key.as_str()))+ .collect();++ let types = etypes+ .iter()+ .filter(|t| picked_class(&t.id))+ .map(|t| (t.key.clone(), t.label.clone(), t.description.clone()))+ .collect();++ // 一侧的类一个都没铺出去就写 `*`:签名是导向,指向看不见的类只会误导+ let sig_of = |ids: &[Uuid]| -> String {+ let mut keys: Vec<&str> = ids+ .iter()+ .filter_map(|id| key_of.get(id).copied())+ .collect();+ if keys.is_empty() {+ return "*".into();+ }+ keys.sort_unstable();+ keys.join("|")+ };+ let relations = rtypes+ .iter()+ .filter(|r| r.kind != "attribute" && r.key != FALLBACK_RELATION_KEY)+ .filter(|r| picked_rel(&r.id))+ .map(|r| {+ let signature = if r.domains.is_empty() && r.ranges.is_empty() {+ String::new()+ } else {+ format!("{} → {}", sig_of(&r.domains), sig_of(&r.ranges))+ };+ utopia_extract::PromptRelation {+ key: r.key.clone(),+ label: r.label.clone(),+ description: r.description.clone(),+ signature,+ }+ })+ .collect();++ let attributes = rtypes+ .iter()+ .filter(|r| r.kind == "attribute" && picked_rel(&r.id))+ .flat_map(|r| r.domains.iter().map(move |d| (r, d)))+ .filter_map(|(r, domain_id)| {+ let class_key = key_of.get(domain_id)?;+ let dt = r.datatype.as_deref().unwrap_or("text");+ let spec = match &r.unit {+ Some(u) if !u.is_empty() => format!("{dt}, {u}"),+ _ => dt.to_string(),+ };+ let d = r.description.trim();+ Some(if d.is_empty() {+ format!("- {class_key}.{} ({spec})", r.key)+ } else {+ format!("- {class_key}.{} ({spec}): {d}", r.key)+ })+ })+ .collect();++ PromptLists {+ types,+ relations,+ attributes,+ }+}++/// 每块检索多少个类 / 关系 / 属性。**待测**——跟预算一样,定它们要那条曲线。+const PER_CHUNK_CLASSES: i64 = 40;+const PER_CHUNK_RELATIONS: i64 = 30;+const PER_CHUNK_ATTRIBUTES: i64 = 30;++/// 按这一块的向量检索候选,排出这一块专用的三段清单。+///+/// 检索失败返回 `Ok(None)` 而不是错误:调用方会退回全量。提示词大是慢,+/// 没有类可选是抽不出东西——两者之间选前者。+async fn chunk_lists(+ state: &AppState,+ kb_id: Uuid,+ embedding: &[f32],+ etypes: &[utopia_core::models::EntityType],+ rtypes: &[utopia_core::models::RelationType],+ seed_classes: &HashSet<Uuid>,+) -> anyhow::Result<Option<PromptLists>> {+ let mut classes: HashSet<Uuid> = seed_classes.clone();+ classes.extend(+ utopia_store::ontology::nearest_entity_type_ids(+ &state.pool,+ kb_id,+ embedding,+ PER_CHUNK_CLASSES,+ )+ .await?,+ );+ let mut rels: HashSet<Uuid> = HashSet::new();+ // 关系与属性分开检索:两段在提示词里是分开的,混在一起取会让其中一段+ // 被另一段挤空+ rels.extend(+ utopia_store::ontology::nearest_relation_type_ids(+ &state.pool,+ kb_id,+ embedding,+ PER_CHUNK_RELATIONS,+ Some("relation"),+ )+ .await?,+ );+ rels.extend(+ utopia_store::ontology::nearest_relation_type_ids(+ &state.pool,+ kb_id,+ embedding,+ PER_CHUNK_ATTRIBUTES,+ Some("attribute"),+ )+ .await?,+ );+ // 一个候选都没检索到 = 索引还没建好,退回全量而不是给一份空清单+ if classes.len() <= seed_classes.len() && rels.is_empty() {+ return Ok(None);+ }+ Ok(Some(build_lists(+ etypes,+ rtypes,+ Some(&classes),+ Some(&rels),+ )))+}+#[cfg(test)]mod tests {use super::looks_literal;diff --git a/crates/utopia-server/src/type_resolution.rs b/crates/utopia-server/src/type_resolution.rsindex 684cd9c1..5d1ddbde 100644--- a/crates/utopia-server/src/type_resolution.rs+++ b/crates/utopia-server/src/type_resolution.rs@@ -1,4 +1,12 @@-//! 类型消解:把挂在倾倒场类下的实体,精化到本体里具体的类。+//! 类型消解:把实体的类型改对——**往细里走,也把走错的掰回来**。+//!+//! 起初只有前半句:抽取给粗类,消解往它的后代精化。抽取开始按分块检索候选+//! 之后,它自己就会挑细类,也就会挑错(实测 `绍兴 → address`、+//! `慢病管理小程序 → entry_point`)。而正确答案是错类的**兄弟**不是它的后代,+//! 于是被"候选必须更细"那条规则挡在门外——那条规则是我自己写进提示词的。+//!+//! 现在两个方向都认。纠正天然判为跨轴,所以一定进人工:推翻抽取的判断比+//! 细化它风险大,不该自动发生。//!//! **为什么能事后做,而谓词不能**(0001 的那个不对称):类型是挂在节点上的//! 注解,可以等证据攒够了再贴;谓词就是事实本身,`(NVIDIA, ?, Mellanox)`@@ -39,6 +47,8 @@ pub struct TypeSuggestion {pub entity_id: Uuid,pub name: String,pub coarse: String,+ /// 现类的描述。裁决要判"现在这个类对不对",光看 key 不够+ pub coarse_description: String,/// 粗类的 id。裁决要拿它跟目标类配成一对,去查"这一对人认可过没有"#[serde(skip)]pub coarse_id: Uuid,@@ -228,6 +238,7 @@ pub async fn preview(state: &AppState, kb_id: Uuid) -> AppResult<Vec<TypeSuggestentity_id: s.id,name: s.canonical_name.clone(),coarse: s.coarse_key.clone(),+ coarse_description: s.coarse_description.clone(),coarse_id: s.coarse_id,proposed_type: s.proposed_type.clone(),specific_type: s.specific_type.clone(),@@ -428,6 +439,10 @@ pub async fn resolve(state: &AppState, kb_id: Uuid) -> AppResult<ResolutionOutco// 不在 = 换了一条分类轴(判成 product 的东西落到了 CreativeWork 底下),// 那是重新分类而不是精化,值得一个人看一眼。实测那一轮唯一明确的错//(《中国数据智能》→ publication_issue)正是这一类。+ //+ // **纠正也走这里,而且天然如此**:抽取挑错细类之后(绍兴 → address),+ // 正确答案是它的兄弟而不是它的后代,所以一定判为跨轴、一定进人工。+ // 这正是想要的——推翻抽取的判断比细化它风险大,不该自动发生。let crosses_axis = !item.descendants.contains(&target.id)&& !approved.contains(&(item.coarse_id, target.id));if confidence >= AUTO_THRESHOLD && !crosses_axis {@@ -476,10 +491,17 @@ pub async fn resolve(state: &AppState, kb_id: Uuid) -> AppResult<ResolutionOutcofn adjudication_prompt(items: &[TypeSuggestion]) -> String {let mut blocks = Vec::new();for it in items {+ // 现类连描述一起给:要判"现在这个类对不对",光看 key 不够——+ // 导入本体的 key 常常自解释不了(`entry_point` 是什么?)+ let current = if it.coarse_description.trim().is_empty() {+ it.coarse.clone()+ } else {+ format!("{} ({})", it.coarse, it.coarse_description.trim())+ };let mut lines = vec![format!("### {}\ncurrently: {}\nthe extractor called it: {}\nseen as: {}",it.name,- it.coarse,+ current,it.specific_type.as_deref().unwrap_or("-"),it.profile.chars().take(200).collect::<String>())];@@ -504,19 +526,30 @@ fn adjudication_prompt(items: &[TypeSuggestion]) -> String {blocks.push(lines.join("\n"));}format!(- "You are refining entity types in a knowledge graph. Each entity below already has a \- broad type and a list of candidate narrower types retrieved from the ontology.\n\+ "You are fixing entity types in a knowledge graph. Each entity below has a type it was \+ given during extraction, and a list of candidate types retrieved from the ontology.\n\+ \n\+ For each entity choose ONE candidate key, or null. There are two reasons to choose \+ a candidate, and they are different:\n\+ \n\+ **Narrowing** — the current type is right but broad, and a candidate says the same \+ thing more precisely (organization → hospital).\n\\n\- For each entity choose ONE candidate key, or null.\n\+ **Correcting** — the current type is simply wrong, and a candidate is right. This \+ happens because extraction picks from a retrieved shortlist and can pick badly: a city \+ typed as an address, an app typed as an entry point. A correcting candidate is \+ usually a sibling of the current type rather than a narrower version of it, so do not \+ withhold it on the grounds that it is not more specific. Say plainly in the reason \+ that the current type is wrong; a person will see this one before it is applied.\n\\n\Choose null whenever any of these hold, and expect null to be a common answer:\n\- no candidate actually means the thing (the list is retrieved by similarity, so it \usually contains near-misses and sometimes contains nothing right at all);\n\- - the candidate is not narrower than what it already has;\n\+ - the current type is already right and no candidate is more precise;\n\- the entity is not a thing of that kind at all — a quantity, a capability, a phrase.\n\- Keeping the broad type loses nothing: the entity and its facts stay exactly as they \- are, merely less specific. Picking a wrong narrow type is worse than picking none, \- because it reads as a decided fact.\n\+ Keeping the current type loses nothing: the entity and its facts stay exactly as they \+ are. Picking a wrong type is worse than picking none, because it reads as a decided \+ fact.\n\\n\confidence is your own 0~1: use above 0.85 only when the candidate's definition \plainly describes this entity, not when it is merely the closest of a weak list.\n\diff --git a/crates/utopia-store/src/access.rs b/crates/utopia-store/src/access.rsindex f047f702..e8bb1de0 100644--- a/crates/utopia-store/src/access.rs+++ b/crates/utopia-store/src/access.rs@@ -260,3 +260,16 @@ pub async fn ensure_jwt_secret(pool: &PgPool, generated: &str) -> AppResult<Stri))})}++/// 本体铺进抽取提示词的字符预算。超了改成按分块检索候选。+///+/// **放设置而不是常量**,是因为定死它需要一条曲线:每个本体规模下,全量内联+/// 与按块检索各测一次,看它们在哪里交叉。要重启一次服务才测得了一档的话,+/// 那条曲线不会有人跑第二遍。+pub async fn ontology_prompt_budget(pool: &PgPool) -> AppResult<usize> {+ let row: Option<(i32,)> =+ sqlx::query_as("SELECT ontology_prompt_budget FROM deployment_settings LIMIT 1")+ .fetch_optional(pool)+ .await?;+ Ok(row.map(|(v,)| v.max(0) as usize).unwrap_or(24_000))+}diff --git a/crates/utopia-store/src/ontology.rs b/crates/utopia-store/src/ontology.rsindex ea8dea45..978dad65 100644--- a/crates/utopia-store/src/ontology.rs+++ b/crates/utopia-store/src/ontology.rs@@ -1035,3 +1035,51 @@ pub async fn adopt_iri_onto_key(.await?;Ok(row.map(|(id,)| id))}++/// 与给定向量最近的若干**类 id**(只回 id,调用方手里已有类的全量数据)。+///+/// 给抽取用:分块向量在抽取循环里本来就有(实体消解在用),拿它检索出这一块+/// 可能用得上的类,只把这些铺进提示词。+pub async fn nearest_entity_type_ids(+ pool: &PgPool,+ kb_id: Uuid,+ embedding: &[f32],+ limit: i64,+) -> AppResult<Vec<Uuid>> {+ let rows: Vec<(Uuid,)> = sqlx::query_as(+ "SELECT id FROM entity_types+ WHERE kb_id = $1 AND embedding IS NOT NULL+ ORDER BY embedding <=> $2+ LIMIT $3",+ )+ .bind(kb_id)+ .bind(Vector::from(embedding.to_vec()))+ .bind(limit)+ .fetch_all(pool)+ .await?;+ Ok(rows.into_iter().map(|(id,)| id).collect())+}++/// 同上,关系与属性。`only_kind` 分道:关系清单与属性清单在提示词里是两段。+pub async fn nearest_relation_type_ids(+ pool: &PgPool,+ kb_id: Uuid,+ embedding: &[f32],+ limit: i64,+ only_kind: Option<&str>,+) -> AppResult<Vec<Uuid>> {+ let rows: Vec<(Uuid,)> = sqlx::query_as(+ "SELECT id FROM relation_types+ WHERE kb_id = $1 AND embedding IS NOT NULL+ AND ($4::text IS NULL OR kind = $4)+ ORDER BY embedding <=> $2+ LIMIT $3",+ )+ .bind(kb_id)+ .bind(Vector::from(embedding.to_vec()))+ .bind(limit)+ .bind(only_kind)+ .fetch_all(pool)+ .await?;+ Ok(rows.into_iter().map(|(id,)| id).collect())+}diff --git a/crates/utopia-store/src/resolution.rs b/crates/utopia-store/src/resolution.rsindex fb8f608f..20ba945d 100644--- a/crates/utopia-store/src/resolution.rs+++ b/crates/utopia-store/src/resolution.rs@@ -1678,9 +1678,13 @@ pub struct TypeCandidateSubject {pub id: Uuid,pub canonical_name: String,pub aliases: Vec<String>,- /// 现在挂着的粗类。**它是地板**:精化只往它的后代走,够不着就原地不动+ /// 现在挂着的类。名字里的"粗"是历史——抽取现在也可能直接给一个细类,+ /// 而且可能给错(实测 `绍兴 → address`),所以这里也可能是要被**纠正**的那个pub coarse_key: String,pub coarse_id: Uuid,+ /// 现类的描述。裁决要判"现在这个类对不对",光看 key 不够——+ /// 导入本体的 key 常常自解释不了(`entry_point` 是什么?)+ pub coarse_description: String,/// 抽取时模型自己报的类型名,**词表里没有**才会留在这儿pub proposed_type: Option<String>,/// 模型对它自己的说法,每个实体都有。@@ -1713,6 +1717,7 @@ pub async fn entities_for_type_resolution() -> AppResult<Vec<TypeCandidateSubject>> {Ok(sqlx::query_as("SELECT e.id, e.canonical_name, e.aliases, t.key AS coarse_key, t.id AS coarse_id,+ coalesce(t.description, '') AS coarse_description,e.proposed_type, e.specific_type,-- 对方写**名字**,不写它的类型 key。-- 写 key 是自毁:查询里出现 organization / product / event 这些词,diff --git a/docs/decisions/0006-ontology-scale-and-the-prompt.md b/docs/decisions/0006-ontology-scale-and-the-prompt.mdnew file mode 100644index 00000000..cfe6be8e--- /dev/null+++ b/docs/decisions/0006-ontology-scale-and-the-prompt.md@@ -0,0 +1,108 @@+# 0006 · 本体规模与抽取提示词++- **状态**:已建成 · 预算与按块检索已上线;预算的具体数值仍待更好的语料来定+- **成文**:2026-08-29(约定见 [README](README.md))++> 这一篇比别的多一样东西:**数字**。它们来自 `scripts/bench/`,任何人都能重跑。+> 每个数字后面都标了它**不能**说明什么——一次运行的差别常常只是跑次方差,+> 而把方差当成效果,是这轮里我犯过不止一次的错。++## 问题++抽取提示词把**整个本体**铺进去,每个文本块重发一遍。`extraction.rs` 从前只有一句+`entity_types(kb)`,一个不落。++导入 schema.org 当前版之后:968 个类、1034 个关系、599 个属性,**每个文本块 108,133+token**。这个数字不是某个设计的代价,**是没有设计的结果**——代码里根本没有"选"这个+动作。0001 的 P3 曾把它归到"800 个类时提示词爆炸",但那时没人量过真实本体,而且+猜错了大头:类清单只占 38%,关系 34%、属性 28%。++## 决定++**按预算切换,不按类数量。**++- 装得下 → 全量铺,行为一字不变。四十个类的本体约 2k 字符,检索反而是多余的往返。+- 装不下 → **每块用它自己的向量检索候选**,内置类恒在,检索不出来就退回全量。++预算按**字符**而不是类数量:类描述的长度差着数量级(`SoftwareApplication` 的描述是+一句话,FIBO 的能有一整段)。判据量的是**实际要排的那段字**(`build_lists` 自己数),+不是另写一个估算公式——公式会跟排版分叉。++分块向量是**现成的**:`chunk.embedding` 在抽取循环里本来就加载了(实体消解在用),+所以按块检索一次嵌入调用都不加。++预算落在 `deployment_settings.ontology_prompt_budget` 而不是常量或环境变量,+理由是测量本身:定它需要每档两组对照,靠重启服务改一档的话,那条曲线不会有人跑第二遍。++### 三个细节只在"选择"存在时才有++1. **签名只能提到铺出去的类。** `works_at (person → organization)` 里那两个 key 必须是+ 模型看得见的——写一个没铺出去的类名,等于教它输出一个不存在的类型。整侧都没选中+ 就退回 `*`。+2. **属性跟着 domain 走。** 属性行是 `class.attr`,它的类没铺出去这行就没意义。+ 这顺带解决了属性段(提示词的 28%)的裁剪,不用单独处理。+3. **两条路共用一个 `build_lists`。** 两份排版代码迟早分叉,而分叉在这里的后果是+ 提示词说的与代码认的不是一回事。++## 量到了什么++同一份语料(`scripts/bench/corpora/pharma.json`,5 篇中文医药文档),**每组一个新库**,+先导本体再灌语料(抽取当场就看得见本体)。++| 内联的类 | 本体段字符 | 模式 | 抽到实体/事实 | 抽取秒 |+|---|---|---|---|---|+| 32 | 2,977 | 全量 | 23 / 25 | 47 |+| 32 | 2,977 | 检索 | 28 / 27 | 31 |+| 78 | 19,187 | 全量 | 25 / 26 | 41 |+| 78 | 19,187 | 检索 | 23 / 26 | 36 |+| 202 | 58,651 | 全量 | 26 / 27 | 36 |+| 202 | 58,651 | 检索 | 23 / 26 | 36 |+| 394 | 169,380 | 全量 | **跑不完(429 TPM 限流)** | — |+| 394 | 169,380 | 检索 | 28 / 26 | 52 |++**一、到 58,651 字符(约 15k token)为止看不到退化。** 实体数在 23–28 之间摆动,+没有趋势。这跟"提示词一大抽取就掉东西"的直觉不符,也跟我先前的一个结论不符+(见下方的撤回)。所以现在的 24,000 字符预算**偏保守**,但改它之前需要更好的语料。++**二、全量内联有一个跟质量无关的硬天花板。** 400 类那一档全量组撞穿了供应商的每分钟+token 限额(429),同一份本体检索组跑得下来。这不是"慢一点",是**跑不完**。++**三、大本体确实带来正确的类型,但要靠检索送到眼前。** 另一组对照(schema.org 全量,+968 类):只有种子本体时 19 个实体里 2 个类型正确;大本体只作用于事后消解时 7 个;+大本体 + 按块检索进提示词时 12 个。++## 这些数字**不能**说明什么++- **每组只跑了一次。** 同一份输入、同一套设置,我们观测到过 25 个实体与 18 个实体的+ 差别。所以实体数那一列的个位数差异**读不出信息**,只有趋势(或没有趋势)可读。+- **子集是按字母序取的。** `subset.mjs` 取前 N 个类,所以 200 类那一档全是 A–B 开头的+ (`AdultEntertainment`、`Brand`…),不含 `hospital`、`pharmacy`、`drug`、`city`。+ 上表**没有列命中率**就是因为这个:正确答案不在子集里,命中涨不上去不是系统的问题。+ 要量"词汇量对质量的贡献",子集得按语料里实际出现的类采样。+- **语料是我们自己写的,标准答案也是自己填的。** 语料、答案、系统三样出自同一只手,+ 过拟合无从检验。这是目前最大的短板,见下。++## 未决++**语料的合法性。** 现在两份语料(科技 / 医药)都是手写的,答案键是逐条人填的。+它们作为 smoke fixture 是称职的——不联网、跑得快、结果确定——但**不足以支撑+"这个改动更好"这种结论**,外部贡献者也无从基于它们做比较。++方向是引入一份外部语料,且**标准答案由外部数据推导而不是由我们填**:中文维基百科+正文(CC BY-SA,可入库可引用)配维基数据的 `P31` 作类型答案。要点不是"维基百科更+真实"——它是百科体,跟企业文档并不一致——而是**答案不再是我们的意见**。+生成脚本必须一并入库,否则"挑哪些条目"又成了一个没人能检查的选择。++**预算与每块候选数的具体值。** 24,000 字符、每块 40 类 / 30 关系 / 30 属性,+都是拍的,代码里标着。上面的曲线说它偏保守,但在语料这一关过掉之前,+按那条曲线去调只是把过拟合调得更紧。++## 撤回++> **「108k 的提示词吃掉了 5 个实体」不成立。** 那条结论来自测量台的默认次序+>(先灌语料再导本体),两组抽取时提示词里都只有 9 个种子类,`prompt_tokens_est`+> 量的是**导入之后**的本体——抽取从没见过它。25 vs 18 是跑次方差。+>+> 台子当时没拦住这个错,因为指标名暗示了顺序没兑现的东西。现在两份规模分开报+>(`ontology_at_extraction` / `ontology_at_resolution`),并有 `--ontology-first`,+> 次序写在输出里而不是靠人记得。diff --git a/docs/decisions/README.md b/docs/decisions/README.mdindex 441282a2..568f44b8 100644--- a/docs/decisions/README.md+++ b/docs/decisions/README.md@@ -27,6 +27,7 @@| 0003 | [本体从语料里长出来,人站在哪一环](0003-ontology-growth-loop.md) | 已建成 · 含三次判断变化的留痕 || 0004 | [语言:哪些字跟着看的人走,哪些跟着语料走](0004-language-and-localization.md) | 已建成 · L0–L3 全部落地 || 0005 | [告警中心](0005-alert-center.md) | 第一刀已建成 · 三个决定推翻两个(就地留痕) |+| 0006 | [本体规模与抽取提示词](0006-ontology-scale-and-the-prompt.md) | 已建成 · 含可重跑的实测数字与一次撤回 |## 不放这里的东西diff --git a/migrations/0042_ontology_prompt_budget.sql b/migrations/0042_ontology_prompt_budget.sqlnew file mode 100644index 00000000..f1468bbb--- /dev/null+++ b/migrations/0042_ontology_prompt_budget.sql@@ -0,0 +1,9 @@+-- 本体铺进抽取提示词的字符预算。超了就改成按分块检索候选。+--+-- 放部署设置而不是环境变量:这一档要能不重启就改。定死它需要一条曲线——+-- 每个本体规模下,全量内联与按块检索各测一次,看它们在哪里交叉。重启一次+-- 服务测一档的话,那条曲线不会有人去跑第二遍。+--+-- 缺省 24000 字符(约 6000 token)是**拍的**,正等着那条曲线来定。+ALTER TABLE deployment_settings+ ADD COLUMN ontology_prompt_budget integer NOT NULL DEFAULT 24000;diff --git a/scripts/bench/run.mjs b/scripts/bench/run.mjsindex f7dacab7..94b234e6 100644--- a/scripts/bench/run.mjs+++ b/scripts/bench/run.mjs@@ -121,22 +121,19 @@ async function main() {psql("UPDATE knowledge_bases SET auto_extend_ontology=FALSE WHERE id='" + kb + "'");await sleep(6000);- const t0 = Date.now();- for (const [filename, content] of corpus.docs) {- await api("POST", "/api/v1/kbs/" + kb + "/ingest", { filename, content });- }- await until(- async () =>- num(- "SELECT count(*) FROM documents WHERE kb_id='" +- kb +- "' AND graph_status='done'",- ) >= corpus.docs.length,- );- const extractMs = Date.now() - t0;+ // **导入与灌语料的先后次序,量的是两件不同的事。**+ //+ // 先灌后导(默认):抽取只看得见种子本体,大本体只作用于事后消解。+ // 先导后灌(--ontology-first):抽取当场就看得见大本体,量的是提示词。+ //+ // 这条曾经害我得出一个错结论:默认次序下报了 108k 的本体段,就说"108k 的+ // 提示词吃掉了 5 个实体"——而那一组抽取时提示词里只有 9 个种子类,两组的+ // 抽取输入根本一样,25 vs 18 是跑次方差。所以下面两处 ontology_size 都记,+ // 各自标明是什么时候量的。+ const ontologyFirst = "ontology-first" in args;- let importMs = 0;- if (args.ontology) {+ async function importOntology() {+ if (!args.ontology) return 0;const t1 = Date.now();const form = new FormData();form.append(@@ -163,28 +160,71 @@ async function main() {10000,2400000,);- importMs = Date.now() - t1;+ return Date.now() - t1;}- const classChars = num(- "SELECT coalesce(sum(length('- '||key||coalesce(': '||nullif(description,''),''))+1),0)" +- " FROM entity_types WHERE kb_id='" +- kb +- "'",- );- const relChars = num(- "SELECT coalesce(sum(length('- '||key||' (x)'||coalesce(': '||nullif(description,''),''))+1),0)" +- " FROM relation_types WHERE kb_id='" +- kb +- "' AND kind<>'attribute'",- );- const attrChars = num(- "SELECT coalesce(sum((length('- '||r.key||' (text)'||coalesce(': '||nullif(r.description,''),''))+1)" +- " * greatest(1,(SELECT count(*) FROM relation_type_domains d WHERE d.relation_type_id=r.id))),0)" +- " FROM relation_types r WHERE r.kb_id='" +- kb +- "' AND r.kind='attribute'",+ const sizeNow = () => {+ const c = num(+ "SELECT coalesce(sum(length('- '||key||coalesce(': '||nullif(description,''),''))+1),0)" ++ " FROM entity_types WHERE kb_id='" ++ kb ++ "'",+ );+ const r = num(+ "SELECT coalesce(sum(length('- '||key||' (x)'||coalesce(': '||nullif(description,''),''))+1),0)" ++ " FROM relation_types WHERE kb_id='" ++ kb ++ "' AND kind<>'attribute'",+ );+ const a = num(+ "SELECT coalesce(sum((length('- '||r.key||' (text)'||coalesce(': '||nullif(r.description,''),''))+1)" ++ " * greatest(1,(SELECT count(*) FROM relation_type_domains d WHERE d.relation_type_id=r.id))),0)" ++ " FROM relation_types r WHERE r.kb_id='" ++ kb ++ "' AND r.kind='attribute'",+ );+ return {+ classes: num("SELECT count(*) FROM entity_types WHERE kb_id='" + kb + "'"),+ relations: num(+ "SELECT count(*) FROM relation_types WHERE kb_id='" + kb + "' AND kind<>'attribute'",+ ),+ attributes: num(+ "SELECT count(*) FROM relation_types WHERE kb_id='" + kb + "' AND kind='attribute'",+ ),+ prompt_chars: c + r + a,+ prompt_tokens_est: Math.round((c + r + a) / CHARS_PER_TOKEN),+ };+ };++ let importMs = 0;+ if (ontologyFirst) importMs = await importOntology();++ // **抽取当时本体有多大**——这一份才是提示词看到的那个规模。+ //+ // 先摸一次本体:种子类是**惰性建**的(第一次读本体或抽取时才落库),+ // 不先摸就量到导入进来那些、漏掉 9 个种子。第一版就漏了,表现是+ // 抽取时 24 类、消解时 32 类,看着像中途有人改了本体+ await api("GET", "/api/v1/kbs/" + kb + "/ontology");+ const atExtraction = sizeNow();++ const t0 = Date.now();+ for (const [filename, content] of corpus.docs) {+ await api("POST", "/api/v1/kbs/" + kb + "/ingest", { filename, content });+ }+ await until(+ async () =>+ num(+ "SELECT count(*) FROM documents WHERE kb_id='" ++ kb ++ "' AND graph_status='done'",+ ) >= corpus.docs.length,);+ const extractMs = Date.now() - t0;++ if (!ontologyFirst) importMs = await importOntology();++ // 消解时本体有多大(先灌后导时它跟抽取当时不同)+ const atResolution = sizeNow();const t2 = Date.now();const outcome = await api("POST", "/api/v1/kbs/" + kb + "/ontology/type-resolution");@@ -241,23 +281,11 @@ async function main() {corpus: corpusName,ontology: args.ontology ? path.basename(args.ontology) : null,kb_id: kb,- ontology_size: {- classes: num("SELECT count(*) FROM entity_types WHERE kb_id='" + kb + "'"),- relations: num(- "SELECT count(*) FROM relation_types WHERE kb_id='" +- kb +- "' AND kind<>'attribute'",- ),- attributes: num(- "SELECT count(*) FROM relation_types WHERE kb_id='" +- kb +- "' AND kind='attribute'",- ),- prompt_chars: classChars + relChars + attrChars,- prompt_tokens_est: Math.round(- (classChars + relChars + attrChars) / CHARS_PER_TOKEN,- ),- },+ order: ontologyFirst ? "ontology-first" : "documents-first",+ // **两份,各自标明什么时候量的。** 只报一份就会被读成"抽取用的提示词+ // 有这么大",而先灌后导时抽取根本没见过它——这个误读已经发生过一次+ ontology_at_extraction: atExtraction,+ ontology_at_resolution: atResolution,graph: {entities: num("SELECT count(*) FROM entities WHERE kb_id='" +diff --git a/scripts/bench/subset.mjs b/scripts/bench/subset.mjsnew file mode 100644index 00000000..6eda6b14--- /dev/null+++ b/scripts/bench/subset.mjs@@ -0,0 +1,76 @@+#!/usr/bin/env node+// 把 schema.org 的 TTL 切成前 N 个类的子集,给退化曲线用。+//+// 曲线要回答的是:**内联多少词汇量之后抽取开始掉东西**。那条曲线定+// `ONTOLOGY_PROMPT_BUDGET` 与每块检索多少个候选,不测就是拍脑袋。+//+// 为什么不用"导入全量再限制内联数":那样量到的是"检索选得准不准",+// 混进了检索的质量。切子集 + 全量内联,量的才是纯粹的规模效应。+//+// 用法:node scripts/bench/subset.mjs /tmp/schemaorg.ttl 100 > /tmp/schemaorg-100.ttl++import fs from "node:fs";++const [, , src, nRaw] = process.argv;+const N = Number(nRaw);+if (!src || !Number.isFinite(N)) {+ console.error("用法: subset.mjs <schemaorg.ttl> <类数>");+ process.exit(2);+}++const text = fs.readFileSync(src, "utf8");+const lines = text.split("\n");++// 前缀块原样保留:切掉它文件就解析不了+const prefixEnd = lines.findIndex((l) => l.startsWith("@prefix") === false && l.trim() && !l.startsWith("#"));+const prefixes = lines.slice(0, prefixEnd).join("\n");++// 按空行分块,但**必须知道自己在不在三引号字符串里**。+//+// 前两版都栽在这上面。按 `/\.\s*$/` 收尾不行:schema.org 的 rdfs:comment 里+// 有以句点结尾的行,块从描述中间被劈开(导入报 `Accountancy is not a valid+// subject`)。改按空行也不行:`"""…"""` 里也有真正的空行,同样劈开+//(`A is not a valid subject`,"A" 是 BreadcrumbList 那段描述的第一个词)。+//+// TTL 里没有词法上下文就切不动这个文件——数一下 `"""` 出现过几次,+// 偶数才算在字符串外面。+const blocks = [];+{+ let cur = [];+ let inLiteral = false;+ for (const line of lines.slice(prefixEnd)) {+ const quotes = (line.match(/"""/g) || []).length;+ if (!inLiteral && quotes % 2 === 0 && !line.trim() && cur.length) {+ blocks.push(cur.join("\n").trim());+ cur = [];+ continue;+ }+ cur.push(line);+ if (quotes % 2 === 1) inLiteral = !inLiteral;+ }+ if (cur.length) blocks.push(cur.join("\n").trim());+}++const subjectOf = (b) => (b.match(/^\s*(\S+)\s+a\s/m) || [])[1] || "";+const isClass = (b) => /\ba\s+rdfs:Class\b/.test(b);+const isProp = (b) => /\ba\s+rdf:Property\b/.test(b);++// 取前 N 个类。**保序而不是随机取**:同一个 N 每次得到同一份子集,+// 两次跑出的差别才归因得到别处+const classes = blocks.filter(isClass);+const keep = new Set(classes.slice(0, N).map(subjectOf).filter(Boolean));++// 属性:domainIncludes 落在保留的类里就留。留下指向被切掉的类的属性没有意义+//——那些 domain 解析不出来,导入时本来就会被跳过+const props = blocks.filter(isProp).filter((b) => {+ const m = b.match(/schema:domainIncludes([^;.]*)/);+ if (!m) return false;+ return m[1]+ .split(",")+ .map((x) => x.trim().replace(/[.;]$/, ""))+ .some((x) => keep.has(x));+});++const kept = blocks.filter((b) => isClass(b) && keep.has(subjectOf(b)));+process.stdout.write(prefixes + "\n\n" + kept.concat(props).join("\n\n") + "\n");+process.stderr.write(`保留 ${kept.length} 个类、${props.length} 个属性\n`);diff --git a/scripts/bench/truth/pharma.json b/scripts/bench/truth/pharma.jsonindex 9b6389c6..763fe386 100644--- a/scripts/bench/truth/pharma.json+++ b/scripts/bench/truth/pharma.json@@ -21,7 +21,7 @@"杭州拱墅区": ["administrative_area"],"心血管健康论坛": ["business_event", "event_series", "conference_event"],"慢病管理小程序": ["software_application", "mobile_application"],- "周敏": [],+ "周敏": ["individual_physician", "physician"],"陈国华": [],"仿制药一致性评价": []}