World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
A chunk keeps its table header (#633)
WaylandYang45953d1ea170
diff --git a/Cargo.lock b/Cargo.lockindex 2a464102..9642ace7 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -475,6 +475,17 @@ dependencies = ["syn 3.0.5",]+[[package]]+name = "bstr"+version = "1.13.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f"+dependencies = [+ "memchr",+ "regex-automata",+ "serde_core",+]+[[package]]name = "bumpalo"version = "3.20.3"@@ -1310,7 +1321,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"dependencies = ["libc",- "windows-sys 0.52.0",+ "windows-sys 0.61.2",][[package]]@@ -1343,6 +1354,17 @@ dependencies = ["pin-project-lite",]+[[package]]+name = "fancy-regex"+version = "0.17.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8"+dependencies = [+ "bit-set",+ "regex-automata",+ "regex-syntax",+]+[[package]]name = "fast-float2"version = "0.2.4"@@ -2687,7 +2709,7 @@ version = "0.50.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"dependencies = [- "windows-sys 0.59.0",+ "windows-sys 0.61.2",][[package]]@@ -2880,7 +2902,7 @@ dependencies = ["oxiri","oxrdf","ryu-js",- "thiserror 1.0.69",+ "thiserror 2.0.20",][[package]]@@ -2892,7 +2914,7 @@ dependencies = ["oxilangtag","oxiri","rand 0.9.5",- "thiserror 1.0.69",+ "thiserror 2.0.20",][[package]]@@ -2905,7 +2927,7 @@ dependencies = ["oxrdf","oxrdfxml","oxttl",- "thiserror 1.0.69",+ "thiserror 2.0.20",][[package]]@@ -2918,7 +2940,7 @@ dependencies = ["oxiri","oxrdf","quick-xml 0.37.5",- "thiserror 1.0.69",+ "thiserror 2.0.20",][[package]]@@ -2931,7 +2953,7 @@ dependencies = ["oxilangtag","oxiri","oxrdf",- "thiserror 1.0.69",+ "thiserror 2.0.20",][[package]]@@ -3293,6 +3315,17 @@ dependencies = ["cc",]+[[package]]+name = "pulldown-cmark"+version = "0.13.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e9f068eba8e7071c5f9511831b44f32c740d5adf574e990f946ddb53db2f314e"+dependencies = [+ "bitflags",+ "memchr",+ "unicase",+]+[[package]]name = "quick-xml"version = "0.37.5"@@ -3376,7 +3409,7 @@ dependencies = ["once_cell","socket2","tracing",- "windows-sys 0.52.0",+ "windows-sys 0.61.2",][[package]]@@ -3707,7 +3740,7 @@ dependencies = ["errno","libc","linux-raw-sys",- "windows-sys 0.52.0",+ "windows-sys 0.61.2",][[package]]@@ -3765,7 +3798,7 @@ dependencies = ["security-framework","security-framework-sys","webpki-root-certs",- "windows-sys 0.52.0",+ "windows-sys 0.61.2",][[package]]@@ -4666,10 +4699,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"dependencies = ["fastrand",- "getrandom 0.3.4",+ "getrandom 0.4.3","once_cell","rustix",- "windows-sys 0.52.0",+ "windows-sys 0.61.2",][[package]]@@ -4696,6 +4729,7 @@ dependencies = ["memchr","strum 0.28.0","thiserror 2.0.20",+ "tiktoken-rs",][[package]]@@ -4747,6 +4781,21 @@ dependencies = ["cfg-if",]+[[package]]+name = "tiktoken-rs"+version = "0.12.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "027853bbf8c7763b77c5c595f1c271c7d536ced7d6f83452911b944621e57fc2"+dependencies = [+ "anyhow",+ "base64 0.22.1",+ "bstr",+ "fancy-regex",+ "lazy_static",+ "regex",+ "rustc-hash",+]+[[package]]name = "time"version = "0.3.55"@@ -5185,9 +5234,11 @@ dependencies = ["oxttl","pdf-extract","percent-encoding",+ "pulldown-cmark","quick-xml 0.42.0","text-splitter","thiserror 2.0.20",+ "tiktoken-rs","tracing","zip 6.0.0",]@@ -5529,7 +5580,7 @@ version = "0.1.11"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"dependencies = [- "windows-sys 0.48.0",+ "windows-sys 0.61.2",][[package]]diff --git a/Cargo.toml b/Cargo.tomlindex 351d61cf..5afaa522 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -60,7 +60,9 @@ utopia-search = { path = "crates/utopia-search" }utopia-llm = { path = "crates/utopia-llm" }tantivy = "0.26"tantivy-jieba = "0.20"-text-splitter = "0.32"+text-splitter = { version = "0.32", features = ["tiktoken-rs"] }+# 分块预算按 token 数:cl100k 的排名表随 crate 内嵌,离线可用+tiktoken-rs = "0.12"pdf-extract = "0.12"calamine = "0.36"pgvector = { version = "0.4", features = ["sqlx"] }diff --git a/crates/utopia-core/src/config.rs b/crates/utopia-core/src/config.rsindex 7ed40ac6..e1af1683 100644--- a/crates/utopia-core/src/config.rs+++ b/crates/utopia-core/src/config.rs@@ -26,6 +26,9 @@ pub struct AppConfig {pub web_dist: String,/// 数据目录:原始文件(files/)与 Tantivy 索引(index/)。pub data_dir: String,+ /// 一个分块的预算(cl100k token)。是个旋钮而不是常量,因为它要能被量:+ /// 结构(表头跟着走)与大小是两件事,召回台子得能只动一个。`UTOPIA_CHUNK_TOKENS`+ pub chunk_tokens: usize,/// 数据库连接池上限。缺省 32,与 worker 并发的缺省对齐——池子小于并发时/// 症状是请求变慢而不是任何一处说"池子不够",所以它必须可调。pub db_max_connections: Option<u32>,@@ -46,6 +49,7 @@ impl Default for AppConfig {secret_key: None,web_dist: "web/dist".into(),data_dir: "data".into(),+ chunk_tokens: 300,db_max_connections: None,cookie_secure: false,open_registration: true,@@ -155,6 +159,7 @@ mod tests {jail.set_env("UTOPIA_BIND_ADDR", "");let cfg = AppConfig::load().unwrap();assert_eq!(cfg.data_dir, "data");+ assert_eq!(cfg.chunk_tokens, 300);assert_eq!(cfg.web_dist, "web/dist");assert_eq!(cfg.bind_addr, "0.0.0.0:1516");Ok(())diff --git a/crates/utopia-ingest/Cargo.toml b/crates/utopia-ingest/Cargo.tomlindex b8dee99c..7b620f68 100644--- a/crates/utopia-ingest/Cargo.toml+++ b/crates/utopia-ingest/Cargo.toml@@ -7,6 +7,8 @@ license.workspace = true[dependencies]anyhow.workspace = truetext-splitter.workspace = true+tiktoken-rs.workspace = true+pulldown-cmark.workspace = truepdf-extract.workspace = truecalamine.workspace = truehtmd.workspace = truediff --git a/crates/utopia-ingest/src/blocks.rs b/crates/utopia-ingest/src/blocks.rsnew file mode 100644index 00000000..ee9fafba--- /dev/null+++ b/crates/utopia-ingest/src/blocks.rs@@ -0,0 +1,297 @@+//! 文档块:把解析器出的 Markdown 读成**顶层块**的序列,每块带着自己在原文里的+//! 字节范围。分块器在这一层之上打包,而不是直接对着字符串找边界。+//!+//! 为什么要这一层:字符串里没有「这是一张表的第三行」这个信息,切分器只能靠+//! 换行和空行猜。猜出来的结果量过(召回台子,2026-09-13):财报 35 块里 13 块+//! 从表格中间开始、没有表头,模型面对一行五个美元数只能猜哪列是哪季度;投票+//! 结果里一张表被原文的分页符劈成两半,后半截成了没有名字的孤表。+//!+//! 这里只做两件事:读出块,和把被分页符劈开的表接回去。切多大、怎么补头,+//! 归 `chunker`。++use pulldown_cmark::{Event, Options, Parser, Tag, TagEnd};+use std::ops::Range;++/// 一个顶层块是什么。范围一律对齐到整行,切出来的文本一个字不改。+#[derive(Debug, Clone, PartialEq, Eq)]+pub enum Kind {+ /// `#` 的个数+ Heading(u8),+ Paragraph,+ /// `head` 是表头行连同分隔行;`rows` 是体行,一行一个范围。+ /// 接回来的续表让 `head`/`rows` 不再连续,所以表的文本一律由这两样拼,+ /// 不从 `Block::range` 整段切+ Table {+ head: Range<usize>,+ rows: Vec<Range<usize>>,+ },+ /// 水平线。在这类文档里多半是**分页符**,不是章节的边界+ Rule,+ /// 列表、代码块、引用、原样 HTML:整块一个单元,不往里看+ Other,+}++#[derive(Debug, Clone, PartialEq, Eq)]+pub struct Block {+ pub kind: Kind,+ pub range: Range<usize>,+}++/// 读出顶层块,并把被分页符劈开的表接回去。+pub fn blocks(text: &str) -> Vec<Block> {+ join_continuations(text, parse(text))+}++/// 正在收的表:整表范围、表头范围、体行范围+type OpenTable = (Range<usize>, Option<Range<usize>>, Vec<Range<usize>>);++fn parse(text: &str) -> Vec<Block> {+ let mut out: Vec<Block> = Vec::new();+ let mut depth = 0usize;+ let mut table: Option<OpenTable> = None;+ for (event, range) in Parser::new_ext(text, Options::ENABLE_TABLES).into_offset_iter() {+ match event {+ Event::Start(tag) => {+ if depth == 0 {+ let kind = match &tag {+ Tag::Heading { level, .. } => Some(Kind::Heading(*level as u8)),+ Tag::Paragraph => Some(Kind::Paragraph),+ Tag::Table(_) => {+ table = Some((range.clone(), None, Vec::new()));+ None+ }+ _ => Some(Kind::Other),+ };+ if let Some(kind) = kind {+ out.push(Block {+ kind,+ range: lines(text, range),+ });+ }+ } else if depth == 1 {+ if let Some((_, head, rows)) = table.as_mut() {+ match tag {+ Tag::TableHead => *head = Some(lines(text, range)),+ Tag::TableRow => rows.push(lines(text, range)),+ _ => {}+ }+ }+ }+ depth += 1;+ }+ Event::End(tag) => {+ depth = depth.saturating_sub(1);+ if depth == 0 && tag == TagEnd::Table {+ if let Some((whole, head, rows)) = table.take() {+ let whole = lines(text, whole);+ // 表头连同它下面的分隔行:从表的开头到第一个体行之前+ let head = match (head, rows.first()) {+ (Some(_), Some(first)) => whole.start..line_before(text, first.start),+ _ => whole.clone(),+ };+ out.push(Block {+ kind: Kind::Table { head, rows },+ range: whole,+ });+ }+ }+ }+ Event::Rule if depth == 0 => out.push(Block {+ kind: Kind::Rule,+ range: lines(text, range),+ }),+ _ => {}+ }+ }+ out+}++/// 被分页符劈开的表接回去。+///+/// 形状很具体:`表 A · 分隔线 · 表 B`,两张一样宽,而 B 的「表头」首格跟 A 某个+/// 体行的首格共享开头两个词——`Number of shares Abstaining` 接在+/// `Number of shares For / Against` 后面。那不是 B 的表头,是 A 的下一行,+/// 解析器只是因为分隔行在它上面才把它当成了表头。+///+/// 判据故意窄:只看行首的标签,不看数。「同宽、表头里有数字」会把两张按年份+/// 分列的财务表接成一张——`2025 | 2024` 也是数字。+fn join_continuations(text: &str, blocks: Vec<Block>) -> Vec<Block> {+ let mut out: Vec<Block> = Vec::with_capacity(blocks.len());+ let mut i = 0usize;+ while i < blocks.len() {+ let b = blocks[i].clone();+ let joined = match (&b.kind, out.last()) {+ (+ Kind::Rule,+ Some(Block {+ kind: Kind::Table { .. },+ ..+ }),+ ) => match blocks.get(i + 1) {+ Some(Block {+ kind: Kind::Table { head, rows },+ range,+ }) if continues(text, out.last().unwrap(), head) => {+ let Some(Block {+ kind:+ Kind::Table {+ rows: prev_rows, ..+ },+ range: prev_range,+ }) = out.last_mut()+ else {+ unreachable!()+ };+ // B 的「表头行」是 A 的一个体行;分隔行不要+ prev_rows.push(first_line(text, head));+ prev_rows.extend(rows.iter().cloned());+ prev_range.end = range.end;+ i += 2;+ true+ }+ _ => false,+ },+ _ => false,+ };+ if !joined {+ out.push(b);+ i += 1;+ }+ }+ out+}++fn continues(text: &str, prev: &Block, next_head: &Range<usize>) -> bool {+ let Kind::Table { head, rows } = &prev.kind else {+ return false;+ };+ let width = |r: &Range<usize>| cell_count(&text[first_line(text, r)]);+ if width(head) != width(next_head) {+ return false;+ }+ let lead = |r: &Range<usize>| {+ first_cell(&text[first_line(text, r)])+ .split_whitespace()+ .take(2)+ .map(str::to_lowercase)+ .collect::<Vec<_>>()+ };+ let next = lead(next_head);+ next.len() == 2 && rows.iter().any(|r| lead(r) == next)+}++/// 范围对齐到整行:往前退到行首,往后走到行尾(不含换行)。+fn lines(text: &str, r: Range<usize>) -> Range<usize> {+ let start = text[..r.start.min(text.len())]+ .rfind('\n')+ .map_or(0, |i| i + 1);+ let mut end = r.end.min(text.len());+ while end > start && matches!(text.as_bytes()[end - 1], b'\n' | b' ' | b'\t') {+ end -= 1;+ }+ let mut end = text[end..].find('\n').map_or(text.len(), |i| end + i);+ // 扩到行尾会把行尾的空格再收回来;块的文本不带尾随空白+ while end > start && matches!(text.as_bytes()[end - 1], b' ' | b'\t') {+ end -= 1;+ }+ start..end+}++/// 上一行的末尾(不含换行):一个范围结束在哪儿,下一段从哪儿起+fn line_before(text: &str, at: usize) -> usize {+ let mut end = at;+ while end > 0 && matches!(text.as_bytes()[end - 1], b'\n' | b' ' | b'\t') {+ end -= 1;+ }+ end+}++fn first_line(text: &str, r: &Range<usize>) -> Range<usize> {+ let end = text[r.start..r.end]+ .find('\n')+ .map_or(r.end, |i| r.start + i);+ r.start..end+}++fn cell_count(line: &str) -> usize {+ let t = line.trim();+ if t.starts_with('|') && t.ends_with('|') && t.len() > 1 {+ t[1..t.len() - 1].split('|').count()+ } else {+ 0+ }+}++fn first_cell(line: &str) -> &str {+ let t = line.trim();+ t.trim_start_matches('|')+ .split('|')+ .next()+ .unwrap_or("")+ .trim()+}++#[cfg(test)]+mod tests {+ use super::*;++ fn kinds(text: &str) -> Vec<String> {+ blocks(text)+ .into_iter()+ .map(|b| match b.kind {+ Kind::Heading(l) => format!("h{l}"),+ Kind::Paragraph => "p".into(),+ Kind::Table { rows, .. } => format!("table({})", rows.len()),+ Kind::Rule => "rule".into(),+ Kind::Other => "other".into(),+ })+ .collect()+ }++ #[test]+ fn a_table_keeps_its_header_and_rows_apart() {+ let text = "Intro.\n\n| a | b |\n| --- | --- |\n| 1 | 2 |\n| 3 | 4 |\n\nAfter.";+ let b = blocks(text);+ assert_eq!(kinds(text), ["p", "table(2)", "p"]);+ let Kind::Table { head, rows } = &b[1].kind else {+ panic!()+ };+ assert_eq!(&text[head.clone()], "| a | b |\n| --- | --- |");+ assert_eq!(&text[rows[0].clone()], "| 1 | 2 |");+ assert_eq!(&text[rows[1].clone()], "| 3 | 4 |");+ assert_eq!(+ &text[b[1].range.clone()],+ "| a | b |\n| --- | --- |\n| 1 | 2 |\n| 3 | 4 |"+ );+ }++ /// 投票结果 8-K 里 Stephen C. Neal 那张表的形状:原文的分页符落在表中间,+ /// 后半截带着一条分隔行,解析器把「弃权」那一行当成了新表的表头。+ #[test]+ fn a_headerless_table_after_a_rule_continues_the_one_before() {+ let text = "| g. Stephen C. Neal | |\n| --- | --- |\n| Number of shares For | 14,573 |\n| Number of shares Against | 2,234 |\n\n* * *\n\n| Number of shares Abstaining | 49,064 |\n| --- | --- |\n| Number of Broker Non-Votes | 2,829 |\n";+ assert_eq!(kinds(text), ["table(4)"]);+ let Kind::Table { rows, .. } = &blocks(text)[0].kind else {+ panic!()+ };+ let rows: Vec<&str> = rows.iter().map(|r| &text[r.clone()]).collect();+ assert_eq!(rows[2], "| Number of shares Abstaining | 49,064 |");+ assert_eq!(rows[3], "| Number of Broker Non-Votes | 2,829 |");+ }++ /// 两张按年份分列的表隔着分页符:表头是 `2025 | 2024` 这种数字,不接。+ #[test]+ fn two_tables_with_numeric_headers_stay_two_tables() {+ let text = "| Revenue | 2025 |\n| --- | --- |\n| Total | 10 |\n\n* * *\n\n| 2024 | 2023 |\n| --- | --- |\n| Total | 9 |\n";+ assert_eq!(kinds(text), ["table(1)", "rule", "table(1)"]);+ }++ #[test]+ fn headings_rules_and_lists_are_their_own_blocks() {+ let text = "# Title\n\nPara one.\n\n- a\n- b\n\n---\n\n## Sub\n\nPara two.";+ assert_eq!(kinds(text), ["h1", "p", "other", "rule", "h2", "p"]);+ let b = blocks(text);+ assert_eq!(&text[b[0].range.clone()], "# Title");+ assert_eq!(&text[b[2].range.clone()], "- a\n- b");+ }+}diff --git a/crates/utopia-ingest/src/chunker.rs b/crates/utopia-ingest/src/chunker.rsindex 09575035..405bad8c 100644--- a/crates/utopia-ingest/src/chunker.rs+++ b/crates/utopia-ingest/src/chunker.rs@@ -1,6 +1,26 @@-//! 分块:text-splitter 语义分块(段落/句子边界优先),字符预算 + 重叠。+//! 分块:把文档块打包成抽取一次能看见的单元。+//!+//! **块的职责是「抽取一次能看见的全部」。**模型只看这一块,切开的东西补不回来,+//! 所以块边界是抽取契约的一部分,不是文本处理的细节。从这条出发定的规矩:+//!+//! - 一行不拆;表头与表体不拆;说明句(表上面那句「结果如下:」)与它的表不拆;+//! 标题与它下面的第一段不拆。+//! - 一张表放得下就整张放。放不下才按行切,**每个续块重复面包屑、说明句和表头**+//! ——这一条才是修「续块只剩一行五个美元数」的那一刀,换再聪明的切分器都不解决它。+//! - 预算按 token 算,不按字符。从前是 1200 字符,注释说「约 1000+ token」,那是中文;+//! 英文只有 300 上下,一张宽表放不下。现在两种语言拿到的是同一个预算。+//! - 水平线**不是**切点。从前的切分器偏爱在水平线上切,而这类文档里水平线是分页符:+//! 投票结果里一张表正是被它劈成了两半。+//! - 不重叠。块是语义单元,上下文靠面包屑和补头给,不靠把上一块的尾巴抄一遍。+//!+//! 每块的正文都是原文的切片,一个字不改;前缀(面包屑、说明句、表头)也是原文的+//! 切片,只是从别处抄来的。`char_start` / `char_end` 指正文那一段,不含前缀。+use crate::blocks::{blocks, Block, Kind};+use std::ops::Range;+use std::sync::OnceLock;use text_splitter::{ChunkConfig, TextSplitter};+use tiktoken_rs::CoreBPE;#[derive(Debug, Clone)]pub struct ChunkPiece {@@ -8,22 +28,691 @@ pub struct ChunkPiece {pub text: String,pub char_start: i32,pub char_end: i32,+ /// 这块所在的章节路径,`›` 分隔;没有标题的文档为 None+ pub heading: Option<String>,}-/// 默认预算:1200 字符(中文约等于 1000+ token),重叠 150。+/// 一块的预算(cl100k token)。+///+/// **300,不是 1000。**1000 是旧字符预算的注释里写的本意,第一轮就量掉了:收购+/// 8-K 从九块变成两块,模型对着 4700 字符只写了七条事实——地址、电话、「published+/// in the SEC」——收购本身一条都没有。模型一次调用写出的事实数不随输入变长而变多,+/// 块一大它就挑最容易的几条写。300 是旧切法在英文上的实际大小,也是 43/45 那两轮+/// 量出来的条件;中文在这个数下拿到的上下文比从前少,还没量(0039 开放问题)。+pub const BUDGET_TOKENS: usize = 300;++/// 说明句最长多少字节还算说明句:表上面那一段要短、或者以冒号结尾,+/// 才当成表的一部分带着走;一整段分析不是说明句+const CAPTION_MAX_BYTES: usize = 200;++/// 不到这么多 token 的一段(页码、脚注标记)不单独成块,并进上一块+const TINY_TOKENS: usize = 8;+pub fn chunk_text(text: &str) -> Vec<ChunkPiece> {- let config = ChunkConfig::new(1200)- .with_overlap(150)- .expect("overlap < capacity");- let splitter = TextSplitter::new(config);- splitter- .chunk_indices(text)+ chunk_with_budget(text, BUDGET_TOKENS)+}++fn bpe() -> &'static CoreBPE {+ static BPE: OnceLock<CoreBPE> = OnceLock::new();+ BPE.get_or_init(|| tiktoken_rs::cl100k_base().expect("cl100k 的排名表随 crate 内嵌"))+}++fn tokens(s: &str) -> usize {+ bpe().encode_ordinary(s).len()+}++/// 打包的单元:标题贴着下一块走;表带着自己的说明句。+enum Unit {+ Heading {+ level: u8,+ range: Range<usize>,+ },+ Text(Range<usize>),+ Table {+ caption: Option<Range<usize>>,+ head: Range<usize>,+ rows: Vec<Range<usize>>,+ },+}++fn units(text: &str, blocks: Vec<Block>) -> Vec<Unit> {+ let mut out: Vec<Unit> = Vec::with_capacity(blocks.len());+ let mut i = 0usize;+ while i < blocks.len() {+ let b = &blocks[i];+ match &b.kind {+ Kind::Heading(level) => out.push(Unit::Heading {+ level: *level,+ range: b.range.clone(),+ }),+ Kind::Rule => {}+ Kind::Paragraph => {+ // 紧跟着表的短段落是说明句,跟表走。**两条通用的补充**(召回台子,+ // 2026-09-13 充值后那一轮量出来的):+ //+ // - 说明句和表之间隔着水平线也算紧跟。这类文档里水平线是分页符,+ // 第五项议案的「结果如下:」和它的表正好被一页隔开,说明句留在了+ // 上一块,表去了下一块。+ // - 一句说明引出的是**紧跟其后的一串表**,不只第一张。「选举十位董事+ // 提名人,结果如下:」后面是十张表,从前只有第一张带着它;黄仁勋+ // 那张在另一块里,四个票数都对,却没有一句话说这是在投什么,+ // 「当选董事」那条边就没了。一串到别的东西(段落、标题)出现为止。+ let caption_like = b.range.len() <= CAPTION_MAX_BYTES+ || text[b.range.clone()].trim_end().ends_with(':');+ let mut j = i + 1;+ let mut attached = 0usize;+ if caption_like {+ loop {+ while matches!(+ blocks.get(j),+ Some(Block {+ kind: Kind::Rule,+ ..+ })+ ) {+ j += 1;+ }+ let Some(Block {+ kind: Kind::Table { head, rows },+ ..+ }) = blocks.get(j)+ else {+ break;+ };+ out.push(Unit::Table {+ caption: Some(b.range.clone()),+ head: head.clone(),+ rows: rows.clone(),+ });+ attached += 1;+ j += 1;+ }+ }+ if attached == 0 {+ out.push(Unit::Text(b.range.clone()));+ } else {+ i = j - 1;+ }+ }+ Kind::Table { head, rows } => out.push(Unit::Table {+ caption: None,+ head: head.clone(),+ rows: rows.clone(),+ }),+ Kind::Other => out.push(Unit::Text(b.range.clone())),+ }+ i += 1;+ }+ out+}++/// 正文里的一项:一段文本,或一张(可能只剩几行的)表+enum Piece {+ Text(Range<usize>),+ Table {+ caption: Option<Range<usize>>,+ head: Range<usize>,+ rows: Vec<Range<usize>>,+ },+}++impl Piece {+ fn render(&self, text: &str) -> String {+ match self {+ Piece::Text(r) => text[r.clone()].to_string(),+ Piece::Table {+ caption,+ head,+ rows,+ } => {+ let mut s = String::new();+ if let Some(c) = caption {+ s.push_str(&text[c.clone()]);+ s.push_str("\n\n");+ }+ s.push_str(&text[head.clone()]);+ for r in rows {+ s.push('\n');+ s.push_str(&text[r.clone()]);+ }+ s+ }+ }+ }+ fn span(&self) -> Range<usize> {+ match self {+ Piece::Text(r) => r.clone(),+ Piece::Table {+ caption,+ head,+ rows,+ } => {+ let start = caption.as_ref().map_or(head.start, |c| c.start);+ let end = rows.last().map_or(head.end, |r| r.end);+ start..end+ }+ }+ }+}++struct Packer<'a> {+ text: &'a str,+ budget: usize,+ out: Vec<ChunkPiece>,+ /// 现在在哪一章:(层级, 标题行)+ path: Vec<(u8, Range<usize>)>,+ /// 出现了、还没贴到正文上的标题+ pending: Vec<Range<usize>>,+ /// 正在攒的块:前缀(章节标题)与正文+ prefix: Vec<Range<usize>>,+ body: Vec<Piece>,+}++impl<'a> Packer<'a> {+ fn render(&self, prefix: &[Range<usize>], body: &[Piece]) -> String {+ let mut parts: Vec<String> = prefix+ .iter()+ .map(|r| self.text[r.clone()].to_string())+ .collect();+ // 同一块里一串表共用一句说明:说明只在第一张前面出现一次。+ // 十位董事的十张表挤在一块里,不该把「结果如下」念十遍+ let mut last_caption: Option<Range<usize>> = None;+ for p in body {+ match p {+ Piece::Table {+ caption: Some(c),+ head,+ rows,+ } if last_caption.as_ref() == Some(c) => {+ let bare = Piece::Table {+ caption: None,+ head: head.clone(),+ rows: rows.clone(),+ };+ parts.push(bare.render(self.text));+ }+ Piece::Table { caption, .. } => {+ last_caption = caption.clone();+ parts.push(p.render(self.text));+ }+ Piece::Text(_) => {+ last_caption = None;+ parts.push(p.render(self.text));+ }+ }+ }+ parts.join("\n\n")+ }++ /// 新块的前缀:现在这一章的标题路径,去掉马上要作为正文出现的那几条+ fn prefix_now(&self) -> Vec<Range<usize>> {+ self.path+ .iter()+ .map(|(_, r)| r.clone())+ .filter(|r| !self.pending.contains(r))+ .collect()+ }++ fn fits(&self, prefix: &[Range<usize>], body: &[Piece]) -> bool {+ tokens(&self.render(prefix, body)) <= self.budget+ }++ /// 试着把一项放进当前块;放不下就先收当前块,再试一次空块。+ ///+ /// **极小的一项不单独成块。**新闻稿末尾有一行页码「4」,按预算它放不进已经+ /// 满了的上一块,于是自己成了一块——一个字符的块,抽取照样为它调一次模型。+ /// 几个 token 的东西并进上一块,超预算那几个 token 无所谓。+ fn place(&mut self, piece: Piece) -> Option<Piece> {+ if self.pending.is_empty() {+ if let Piece::Text(r) = &piece {+ if tokens(&self.text[r.clone()]) <= TINY_TOKENS {+ if !self.body.is_empty() {+ self.body.push(piece);+ return None;+ }+ // 上一块已经发出去了(长段落走退路切分会立刻 flush):接到它尾上+ if let Some(last) = self.out.last_mut() {+ last.text.push_str(+ "++",+ );+ last.text.push_str(&self.text[r.clone()]);+ last.char_end = last.char_end.max(r.end as i32);+ return None;+ }+ }+ }+ }+ let mut with_headings: Vec<Piece> = self.pending.iter().cloned().map(Piece::Text).collect();+ with_headings.push(piece);+ if self.body.is_empty() {+ self.prefix = self.prefix_now();+ }+ let mut candidate: Vec<Piece> = std::mem::take(&mut self.body);+ candidate.extend(with_headings);+ if self.fits(&self.prefix.clone(), &candidate) {+ self.body = candidate;+ self.pending.clear();+ return None;+ }+ // 放不下:把刚加的拆回来+ let n = candidate.len() - (self.pending.len() + 1);+ let mut rest = candidate.split_off(n);+ self.body = candidate;+ let piece = rest.pop().expect("刚放进去的那一项");+ if !self.body.is_empty() {+ self.flush();+ return self.place(piece);+ }+ Some(piece)+ }++ fn flush(&mut self) {+ if self.body.is_empty() {+ return;+ }+ let body = std::mem::take(&mut self.body);+ let prefix = std::mem::take(&mut self.prefix);+ let text = self.render(&prefix, &body);+ let start = body.iter().map(|p| p.span().start).min().unwrap_or(0);+ let end = body.iter().map(|p| p.span().end).max().unwrap_or(0);+ let heading = self.breadcrumb();+ self.out.push(ChunkPiece {+ seq: self.out.len() as i32,+ text,+ char_start: start as i32,+ char_end: end as i32,+ heading,+ });+ }++ fn breadcrumb(&self) -> Option<String> {+ if self.path.is_empty() {+ return None;+ }+ Some(+ self.path+ .iter()+ .map(|(_, r)| self.text[r.clone()].trim_start_matches('#').trim())+ .collect::<Vec<_>>()+ .join(" › "),+ )+ }++ fn heading(&mut self, level: u8, range: Range<usize>) {+ while self.path.last().is_some_and(|(l, _)| *l >= level) {+ self.path.pop();+ }+ self.path.push((level, range.clone()));+ self.pending.push(range);+ }++ fn text_unit(&mut self, range: Range<usize>) {+ let Some(Piece::Text(range)) = self.place(Piece::Text(range)) else {+ return;+ };+ // 一段就超预算:退回按句切,每一小段自成一块,前缀照给+ self.prefix = self.prefix_now();+ let prefix_cost = tokens(&self.render(&self.prefix.clone(), &[]));+ let headings: Vec<Piece> = self.pending.drain(..).map(Piece::Text).collect();+ let heading_cost = tokens(&self.render(&[], &headings));+ let capacity = self+ .budget+ .saturating_sub(prefix_cost + heading_cost)+ .max(1);+ let config = ChunkConfig::new(capacity).with_sizer(bpe());+ let splitter = TextSplitter::new(config);+ let block = &self.text[range.clone()];+ let mut first = true;+ for (offset, piece) in splitter.chunk_indices(block) {+ let sub = range.start + offset..range.start + offset + piece.len();+ if first {+ self.body = headings+ .iter()+ .map(|h| match h {+ Piece::Text(r) => Piece::Text(r.clone()),+ _ => unreachable!(),+ })+ .collect();+ first = false;+ }+ self.body.push(Piece::Text(sub));+ self.prefix = self.prefix_now();+ self.flush();+ }+ }++ fn table_unit(+ &mut self,+ caption: Option<Range<usize>>,+ head: Range<usize>,+ rows: Vec<Range<usize>>,+ ) {+ let whole = Piece::Table {+ caption: caption.clone(),+ head: head.clone(),+ rows: rows.clone(),+ };+ let Some(_) = self.place(whole) else {+ return;+ };+ // 整张放不下:按行切,每一块都带说明句和表头。一行不拆;+ // 说明句 + 表头 + 一行还超预算的,照样出一块——没有更好的切法+ let mut rows = rows.into_iter().peekable();+ while rows.peek().is_some() {+ let mut taken: Vec<Range<usize>> = vec![rows.next().expect("peek 过")];+ while let Some(next) = rows.peek() {+ let mut trial = taken.clone();+ trial.push(next.clone());+ let piece = Piece::Table {+ caption: caption.clone(),+ head: head.clone(),+ rows: trial,+ };+ let headings: Vec<Piece> = self.pending.iter().cloned().map(Piece::Text).collect();+ let mut body = headings;+ body.push(piece);+ if self.fits(&self.prefix_now(), &body) {+ taken.push(rows.next().expect("peek 过"));+ } else {+ break;+ }+ }+ self.prefix = self.prefix_now();+ self.body = self.pending.drain(..).map(Piece::Text).collect();+ self.body.push(Piece::Table {+ caption: caption.clone(),+ head: head.clone(),+ rows: taken,+ });+ self.flush();+ }+ }+}++pub fn chunk_with_budget(text: &str, budget: usize) -> Vec<ChunkPiece> {+ let mut p = Packer {+ text,+ budget,+ out: Vec::new(),+ path: Vec::new(),+ pending: Vec::new(),+ prefix: Vec::new(),+ body: Vec::new(),+ };+ for unit in units(text, blocks(text)) {+ match unit {+ Unit::Heading { level, range } => p.heading(level, range),+ Unit::Text(range) => p.text_unit(range),+ Unit::Table {+ caption,+ head,+ rows,+ } => p.table_unit(caption, head, rows),+ }+ }+ // 文档以标题收尾:标题自己成一块,总好过丢掉+ if !p.pending.is_empty() {+ p.prefix = p.prefix_now();+ p.body = p.pending.drain(..).map(Piece::Text).collect();+ }+ p.flush();+ p.out+}++#[cfg(test)]+mod tests {+ use super::*;++ fn table(n_rows: usize) -> String {+ let mut s = String::from("The results of the voting were as follows:\n\n| Nominee | For | Against |\n| --- | --- | --- |\n");+ for i in 0..n_rows {+ s.push_str(&format!("| Director {i} | {} | {} |\n", 1000 + i, 50 + i));+ }+ s+ }++ /// 财报里那 13 块的形状:表比预算大,续块必须还看得见表头和说明句。+ #[test]+ fn a_wide_table_is_split_by_rows_and_every_piece_carries_its_header_and_caption() {+ let text = format!("# Item 5.07\n\n{}", table(40));+ let pieces = chunk_with_budget(&text, 120);+ assert!(+ pieces.len() > 2,+ "40 行在 120 token 里必然切成好几块: {}",+ pieces.len()+ );+ for (i, p) in pieces.iter().enumerate() {+ assert!(+ p.text.contains("| Nominee | For | Against |"),+ "第 {i} 块没有表头:\n{}",+ p.text+ );+ assert!(+ p.text.contains("| --- | --- | --- |"),+ "第 {i} 块没有分隔行"+ );+ assert!(+ p.text+ .contains("The results of the voting were as follows:"),+ "第 {i} 块没有说明句"+ );+ assert!(+ p.text.starts_with("# Item 5.07"),+ "第 {i} 块没有章节面包屑:\n{}",+ p.text+ );+ assert_eq!(p.heading.as_deref(), Some("Item 5.07"));+ }+ // 每一行只出现一次,一行不拆+ let all: String = pieces+ .iter()+ .map(|p| p.text.as_str())+ .collect::<Vec<_>>()+ .join("\n");+ for i in 0..40 {+ assert_eq!(+ all.matches(&format!("| Director {i} |")).count(),+ 1,+ "Director {i}"+ );+ }+ }++ #[test]+ fn a_table_that_fits_stays_whole_with_its_caption() {+ let text = format!("Intro paragraph.\n\n{}\nClosing.", table(3));+ let pieces = chunk_with_budget(&text, 1000);+ assert_eq!(pieces.len(), 1);+ let t = &pieces[0].text;+ assert!(t.contains("The results of the voting were as follows:\n\n| Nominee"));+ assert!(t.ends_with("Closing."));+ }++ /// 说明句已经在上一块的尾巴上、表放不进去了:说明句得跟着表走,不能留在上一块。+ #[test]+ fn a_caption_never_parts_from_its_table() {+ let filler = "Words about nothing in particular. ".repeat(12);+ let text = format!("{filler}\n\n{}", table(4));+ let pieces = chunk_with_budget(&text, 140);+ assert!(pieces.len() >= 2);+ let with_table = pieces+ .iter()+ .find(|p| p.text.contains("| Nominee |"))+ .expect("有表的那块");+ assert!(with_table+ .text+ .contains("The results of the voting were as follows:"));+ let before: Vec<&ChunkPiece> = pieces+ .iter()+ .filter(|p| !p.text.contains("| Nominee |"))+ .collect();+ for p in before {+ assert!(+ !p.text.contains("were as follows"),+ "说明句留在了没有表的块里:\n{}",+ p.text+ );+ }+ }++ #[test]+ fn a_chunk_opens_with_the_headings_it_lives_under() {+ let text = "# Report\n\n## Part A\n\nFirst.\n\n## Part B\n\nSecond.\n\n### B.1\n\nThird.";+ let pieces = chunk_with_budget(text, 1000);+ assert_eq!(pieces.len(), 1);+ assert!(pieces[0]+ .text+ .starts_with("# Report\n\n## Part A\n\nFirst."));+ assert_eq!(pieces[0].heading.as_deref(), Some("Report › Part B › B.1"));++ let pieces = chunk_with_budget(text, 14);+ let third = pieces.iter().find(|p| p.text.contains("Third.")).unwrap();+ assert!(+ third.text.starts_with("# Report\n\n## Part B\n\n### B.1"),+ "{}",+ third.text+ );+ assert_eq!(third.heading.as_deref(), Some("Report › Part B › B.1"));+ assert_eq!(+ &text[third.char_start as usize..third.char_end as usize],+ "### B.1\n\nThird."+ );+ }++ #[test]+ fn a_long_paragraph_still_splits_by_sentence_and_offsets_are_verbatim() {+ let text = "One sentence here. Another sentence follows it. ".repeat(30);+ let pieces = chunk_with_budget(&text, 40);+ assert!(pieces.len() > 1);+ for p in &pieces {+ assert_eq!(&text[p.char_start as usize..p.char_end as usize], p.text);+ assert!(tokens(&p.text) <= 40);+ }+ }++ #[test]+ fn chunks_are_numbered_from_zero_and_never_empty() {+ let text = "# T\n\nA.\n\n* * *\n\nB.\n\n| x | y |\n| --- | --- |\n| 1 | 2 |";+ let pieces = chunk_text(text);+ assert_eq!(pieces.len(), 1);+ for (i, p) in pieces.iter().enumerate() {+ assert_eq!(p.seq as usize, i);+ assert!(!p.text.trim().is_empty());+ }+ assert!(!pieces[0].text.contains("* * *"), "水平线不进块");+ }++ /// 新闻稿末尾的页码「4」:不单独成块,并进上一块,哪怕上一块已经满了。+ #[test]+ fn a_page_number_never_becomes_a_chunk_of_its_own() {+ let body = "A sentence that fills the budget nicely. ".repeat(6);+ let text = format!(+ "{body}++4"+ );+ let pieces = chunk_with_budget(&text, tokens(body.trim()));+ assert_eq!(+ pieces.len(),+ 1,+ "{:?}",+ pieces.iter().map(|p| &p.text).collect::<Vec<_>>()+ );+ assert!(pieces[0].text.ends_with(+ "++4"+ ));+ }++ /// 第五项议案的形状:说明句 · 分页符 · 表。分页符不隔开说明句和它的表。+ #[test]+ fn a_page_break_between_a_caption_and_its_table_does_not_part_them() {+ let filler = "Words about nothing in particular. ".repeat(10);+ let text = format!(+ "{filler}\n\n5. Stockholders did not approve the proposal. The results of the voting were as follows:\n\n* * *\n\n| Number of shares For | 144 |\n| --- | --- |\n| Number of shares Against | 16 |\n"+ );+ let pieces = chunk_with_budget(&text, 110);+ let table = pieces+ .iter()+ .find(|p| p.text.contains("| Number of shares For |"))+ .expect("有表的那块");+ assert!(+ table+ .text+ .contains("The results of the voting were as follows:"),+ "{}",+ table.text+ );+ for p in pieces+ .iter()+ .filter(|p| !p.text.contains("| Number of shares For |"))+ {+ assert!(+ !p.text.contains("were as follows"),+ "说明句留在了没有表的块里:\n{}",+ p.text+ );+ }+ }++ /// 十位董事的形状:一句说明、一串表。每张表不论落在哪一块都看得见那句说明;+ /// 同一块里的几张表,说明只念一遍。+ #[test]+ fn one_caption_introduces_the_whole_run_of_tables_after_it() {+ let mut text = String::from("Stockholders elected each of the director nominees. The results of the voting were as follows:\n\n");+ for (i, name) in [+ "Tench Coxe",+ "John Dabiri",+ "Jen-Hsun Huang",+ "Dawn Hudson",+ "Harvey Jones",+ ]+ .iter().enumerate()- .map(|(i, (offset, chunk))| ChunkPiece {- seq: i as i32,- text: chunk.to_string(),- char_start: offset as i32,- char_end: (offset + chunk.len()) as i32,- })- .collect()+ {+ text.push_str(&format!(+ "| {name} | |\n| --- | --- |\n| Number of shares For | {} |\n| Number of shares Against | {} |\n\n",+ 1000 + i,+ 50 + i+ ));+ }+ text.push_str(+ "Another matter entirely.\n\n| Unrelated | 1 |\n| --- | --- |\n| row | 2 |\n",+ );+ let pieces = chunk_with_budget(&text, 90);+ assert!(pieces.len() >= 3, "{}", pieces.len());+ let huang = pieces+ .iter()+ .find(|p| p.text.contains("Jen-Hsun Huang"))+ .expect("黄仁勋那块");+ assert!(+ huang.text.contains("director nominees"),+ "续块丢了说明句:\n{}",+ huang.text+ );+ for p in &pieces {+ assert!(+ p.text.matches("were as follows").count() <= 1,+ "同一块里说明念了不止一遍:\n{}",+ p.text+ );+ }+ // 一串到别的东西出现为止:后面那张无关的表不带这句说明+ let unrelated = pieces+ .iter()+ .find(|p| p.text.contains("| Unrelated |"))+ .unwrap();+ let before = &unrelated.text[..unrelated.text.find("| Unrelated |").unwrap()];+ assert!(!before.ends_with("follows:\n\n"), "{}", unrelated.text);+ assert!(unrelated.text.contains("Another matter entirely."));+ }++ #[test]+ fn a_rule_is_not_a_boundary() {+ let text = "Alpha.\n\n* * *\n\nBeta.";+ assert_eq!(chunk_with_budget(text, 1000).len(), 1);+ }}diff --git a/crates/utopia-ingest/src/lib.rs b/crates/utopia-ingest/src/lib.rsindex 58e8f68a..c05b9715 100644--- a/crates/utopia-ingest/src/lib.rs+++ b/crates/utopia-ingest/src/lib.rs@@ -105,12 +105,13 @@ mod html_tests {}}+mod blocks;mod chunker;pub mod html;pub mod ontology_rdf;mod parsers;-pub use chunker::{chunk_text, ChunkPiece};+pub use chunker::{chunk_text, chunk_with_budget, ChunkPiece, BUDGET_TOKENS};/// Decode fetched text with the same encoding detection as file ingestion.pub use parsers::plain_text as decode_text;diff --git a/crates/utopia-server/src/pipeline.rs b/crates/utopia-server/src/pipeline.rsindex 56ce766b..590a0f30 100644--- a/crates/utopia-server/src/pipeline.rs+++ b/crates/utopia-server/src/pipeline.rs@@ -65,7 +65,7 @@ async fn run(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {let text_len = parsed.text.chars().count() as i32;// 2. 分块 + 入库- let pieces = utopia_ingest::chunk_text(&parsed.text);+ let pieces = utopia_ingest::chunk_with_budget(&parsed.text, state.chunk_tokens);let chunk_pairs =utopia_store::documents::replace_chunks(&state.pool, doc.kb_id, document_id, &pieces).await?;diff --git a/crates/utopia-server/src/state.rs b/crates/utopia-server/src/state.rsindex f74f0041..24c37c9e 100644--- a/crates/utopia-server/src/state.rs+++ b/crates/utopia-server/src/state.rs@@ -29,6 +29,8 @@ pub struct AppState {pub open_registration: bool,/// 强制 Secure cookie(配置项);未强制时按请求的 X-Forwarded-Proto 逐次判定pub cookie_secure: bool,+ /// 分块预算,见 `AppConfig::chunk_tokens`+ pub chunk_tokens: usize,/// worker 并发数:调度循环每轮热读——系统设置改动即时生效pub worker_concurrency: Arc<std::sync::atomic::AtomicUsize>,/// 按模型的并发闸门:后台任务调 LLM 前取许可。限额存库,改完即时生效@@ -58,6 +60,7 @@ impl AppState {blob,open_registration: cfg.open_registration,cookie_secure: cfg.cookie_secure,+ chunk_tokens: cfg.chunk_tokens,worker_concurrency: Arc::new(std::sync::atomic::AtomicUsize::new(32)),model_gates: Arc::new(crate::llm_util::ModelGates::default()),events,diff --git a/crates/utopia-store/src/documents.rs b/crates/utopia-store/src/documents.rsindex d48597be..652e9214 100644--- a/crates/utopia-store/src/documents.rs+++ b/crates/utopia-store/src/documents.rs@@ -1024,7 +1024,8 @@ pub async fn replace_chunks(let claimed = claim_pool.get_mut(&piece.text).and_then(|ids| ids.pop());if let Some(id) = claimed {sqlx::query(- "UPDATE chunks SET seq = $2, char_start = $3, char_end = $4, doc_version = $5+ "UPDATE chunks SET seq = $2, char_start = $3, char_end = $4, doc_version = $5,+ heading = $6WHERE id = $1",).bind(id)@@ -1032,6 +1033,7 @@ pub async fn replace_chunks(.bind(piece.char_start).bind(piece.char_end).bind(version)+ .bind(&piece.heading).execute(&mut *tx).await?;adopted.push(id);@@ -1062,8 +1064,8 @@ pub async fn replace_chunks(for (id, piece) in to_insert {sqlx::query("INSERT INTO chunks- (id, kb_id, document_id, seq, text, char_start, char_end, doc_version)- VALUES ($1, $2, $3, $4, $5, $6, $7, $8)",+ (id, kb_id, document_id, seq, text, char_start, char_end, doc_version, heading)+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)",).bind(id).bind(kb_id)@@ -1073,6 +1075,7 @@ pub async fn replace_chunks(.bind(piece.char_start).bind(piece.char_end).bind(version)+ .bind(&piece.heading).execute(&mut *tx).await?;}diff --git a/crates/utopia-store/tests/a_search_reads_the_base_as_it_was.rs b/crates/utopia-store/tests/a_search_reads_the_base_as_it_was.rsindex 8dfc5679..0fcf450f 100644--- a/crates/utopia-store/tests/a_search_reads_the_base_as_it_was.rs+++ b/crates/utopia-store/tests/a_search_reads_the_base_as_it_was.rs@@ -106,6 +106,7 @@ async fn seed(pool: &PgPool) -> anyhow::Result<Fixture> {text: "The 2026 handbook says the office moved to Shenzhen.".into(),char_start: 0,char_end: 51,+ heading: None,}];utopia_store::documents::replace_chunks(pool, kb, doc, &pieces).await?;let new_chunk: (Uuid,) = sqlx::query_as(diff --git a/docs/decisions/0039-a-chunk-is-what-extraction-sees.md b/docs/decisions/0039-a-chunk-is-what-extraction-sees.mdnew file mode 100644index 00000000..16fb6973--- /dev/null+++ b/docs/decisions/0039-a-chunk-is-what-extraction-sees.md@@ -0,0 +1,240 @@+# 0039 · A chunk is what extraction sees++- **Status**: Implemented, cut 1 · the parser's Markdown is read into top-level blocks+ (`blocks.rs`), the packer works over blocks with a token budget (`chunker.rs`): a table+ travels with its caption and header, a table too wide for one chunk is split by rows and every+ continuation repeats caption and header, headings become a breadcrumb prefix and the+ `chunks.heading` column, a table split by a page break is joined back · not in this cut:+ an external parser behind the block model (cut 2, Docling for PDF layout and scans), evidence+ that points at a table cell or an image region, a budget measured for its own sake+- **Written**: 2026-09-13 (conventions in the [README](README.md))+- **Related**: [0006](0006-ontology-scale-and-the-prompt.md) sets the other half of the prompt+ budget; [0033](0033-rss-source-summaries-are-source-scoped.md) is the recall bench's home;+ the recall bench (`scripts/bench/recall.mjs`) is the instrument this record was measured with++> Four SEC filings, a 52-item truth table read off the originals by a person. Two rounds on+> `dev` on 2026-09-13 scored 43 and 45. Relation edges: 12/12 both times. Every miss was a+> literal — a price, a date, a title, a vote count — and the misses barely overlapped between+> rounds. The extractor was not failing to read the sentences; it was being handed tables it+> could not read.++## The problem++The chunker was twenty-nine lines: a character budget of 1200, an overlap of 150, boundaries+chosen by a generic text splitter that knows about paragraphs and sentences and nothing else.+Three things followed, all visible in the chunks stored for the bench corpus:++**A table's continuation lost its header.** Of the earnings release's 35 chunks, 13 began in the+middle of a table. The chunk that holds the non-GAAP EPS opens with `| | Non-GAAP* | | $ | 2.22 |+| | $ | 1.87 | …` — five dollar figures, no column names. Which quarter is $2.22? The model+guesses. Sometimes it guesses right, which is why that item was a miss in one round and a hit+in the next, and why the score moved by two between rounds with the same code.++**A table lost its caption.** "The results of the voting were as follows:" sat at the end of one+chunk and the votes it introduced started the next. The sentence that says what the numbers are+was not in the chunk with the numbers.++**A page break split a table, and the splitter took the break as a boundary.** The 8-K's HTML has+an `<hr>` where the paper page turned, mid-table for Stephen C. Neal's votes. The converter+renders it as a thematic break, the splitter prefers thematic breaks as split points, and the+second half — abstentions and broker non-votes — became a table with no name on it.++Underneath all three: the budget said 1200 characters and the comment beside it said "about+1000+ tokens". That is true of Chinese. English is about 300 tokens in 1200 characters, so a+wide table never fit, and the same document got a quarter of the context in one language that+it got in the other.++## Dead ends++**A smarter splitter alone.** `text-splitter` has a `MarkdownSplitter` that ranks headings above+paragraphs above sentences. It was the obvious first move and it does not do the thing that+matters: it ranks a table row at the same level as a table, so a table that exceeds the budget+is split between rows exactly as before, and nothing repeats the header. It also keeps thematic+breaks as a preferred boundary, which is the Neal failure. What it does give — heading+attachment, token sizing — this cut takes, but as a fallback for one oversized paragraph, not+as the chunker.++**Overlap as context.** The old 150-character overlap was meant to carry context across a+boundary. For prose it carries the tail of a sentence; for a table it carries the tail of the+previous row, which is the wrong information. Context is a header and a caption, and those have+to be chosen, not copied from wherever the previous chunk happened to end.++**An external document parser now.** Docling and Unstructured parse a document into structured+elements and their chunkers repeat table headers on continuation. For HTML, Markdown, DOCX and+spreadsheets our parser already produces the structure; adding a Python service to a Rust+pipeline for a rule we can write in fifty lines buys nothing here. Where those parsers earn+their cost is PDF layout and scanned pages, and that is cut 2 — behind the same block model,+so the chunker does not change when the parser does.++**Chunking the parser's output as blocks, without the join.** Reading the Markdown into blocks+fixes the header and caption. It does not fix Neal: the parser had already produced two tables,+the second with a separator row under a row of data, and a block reader faithfully reports two+tables. The join is a separate rule and it is deliberately narrow (below).++## Decisions++### 1. A chunk is what extraction sees, so its boundary is part of the extraction contract++The model is handed one chunk and nothing else; what the boundary cut away cannot be recovered+by the prompt. So the chunker's rules are written from the extractor's side: never split a table+row; never separate a table's header from its body; never separate a caption from its table;+never separate a heading from the first block under it. These are not typesetting preferences.+Each one, when broken, produced a specific miss in the bench.++### 2. The parser's text is read into blocks, and the packer works over blocks++`blocks.rs` reads the parser's Markdown with `pulldown-cmark` into a sequence of top-level+blocks — heading, paragraph, table (header lines and body rows as separate ranges), rule, other+— each carrying its byte range in the source. `chunker.rs` packs blocks. The split into two+layers is what makes cut 2 cheap: an external parser produces blocks and the packer does not+know or care.++Chunk text is assembled from verbatim slices of the source. Nothing is rewritten, so a quote the+model gives back is still found in the chunk by `span_in_quote`.++### 3. A table travels with its caption and its header, and a continuation repeats both++A short paragraph immediately before a table — at most 200 bytes, or ending in a colon — is the+table's caption and moves with it. A table that fits goes in whole. One that does not is split+between rows, and every piece is caption, header lines, then as many rows as fit. A row is never+split; a piece that is caption plus header plus one row and still over budget is emitted anyway,+because there is no better cut. This is the change that addresses the 13 headerless chunks+directly. The rest of this record makes it possible; this is the point.++> **Revised 2026-09-13, after the funded 300-token round.** Two gaps in "a caption moves with its+> table", both visible in one filing. *A caption introduces the run of tables after it, not the+> first one.* "Stockholders approved the election of each of our ten director nominees. The+> results of the voting were as follows:" is followed by ten tables; the caption travelled with+> Tench Coxe's, and Jen-Hsun Huang's table sat in another chunk with four correct vote counts and+> nothing saying what the vote was for — the "elected director" edge went. A caption now attaches+> to every table that follows it until something other than a table or a rule appears, and within+> one chunk it is rendered once. *A rule between a caption and its table does not part them.*+> Proposal 5's "results were as follows:" and its table are a page apart in the HTML; the lookahead+> saw the rule and not the table, and the caption stayed in the previous chunk. Rules are skipped+> when looking for a caption's tables. Both are rules about document structure, not about SEC+> wording.++### 4. The budget is tokens, not characters — and it is 300, because 1000 was measured and lost++> **Revised 2026-09-13, the same day, by the first bench round.** The paragraph below chose+> 1000 to honour the old comment. One round at 1000 scored 36/52 against 43 and 45 before, and+> the acquisition 8-K showed why: nine chunks became two, and from 4,700 characters the model+> wrote seven facts — the address, the phone number, "published in the SEC" — and not one about+> the acquisition. The extractor's output per call does not grow with its input; handed more, it+> picks the easiest few. The default is now 300, which is what the old character budget came to+> in English and the condition the 43/45 rounds were measured under. Chinese gets less context+> than it did; that is unmeasured and open. The budget is a knob (`UTOPIA_CHUNK_TOKENS`) so the+> next measurement does not need a rebuild.++Counted with `tiktoken` cl100k, which is embedded in the crate and needs no network. It is not+the extraction model's tokenizer — that is DeepSeek's — but it is a BPE with a similar ratio,+and the budget is a ceiling, not an accounting. 1000 is the number the old comment said the+character budget was meant to be. Chinese keeps roughly the context it had; English gets three+times as much, which is what lets a wide table fit at all.++Budget size is a knob with its own trade-off — fewer, larger calls against more facts per reply+— and it was not measured on its own in this cut. The bench comparison below therefore compares+structure and budget together. Measuring the budget alone is open.++### 5. Headings are a breadcrumb, written into the chunk and into `chunks.heading`++The heading path a chunk sits under is prepended to its text, as the heading lines themselves,+and stored as plain text in `chunks.heading`, a column that had existed since the ingest+migration and was never written. The prefix is skipped when the chunk's body already begins with+that heading. A heading never ends a chunk on its own; it attaches to what follows.++### 6. A thematic break is not a boundary, and a table split by one is joined back++The old splitter preferred to break at a horizontal rule. In this corpus a horizontal rule is a+page break, and a page break is where a table is most likely to be cut in half. Rules are dropped+from the block sequence and do not influence packing.++Separately, in `blocks.rs`: the sequence *table, rule, table* where the two tables have the same+column count and the second's "header" row begins with the same two words as one of the first's+body rows — `Number of shares Abstaining` after `Number of shares For` — is one table. The+second's header row becomes a body row of the first, and its separator line is dropped. The rule+is narrow on purpose. A wider one, "same width and the header holds numbers", would join two+financial tables whose headers are years, because `2025 | 2024` are numbers too. A test holds+both shapes.++### 7. No overlap++Blocks are semantic units. Context is the breadcrumb, the caption and the header, chosen for+what they are, not the tail of the previous chunk copied for where it happened to end. Overlap+also doubled the facts the extractor was asked to produce on the overlapped span; the ledger+deduplicates, but the calls were paid for.++### 8. `char_start` and `char_end` name the body, not the prefix++The two columns point at the verbatim span of the chunk's body in the parsed text; the prefix+(breadcrumb, repeated caption, repeated header) is copied from elsewhere and is not part of that+span. Nothing reads these columns today except one test fixture; this is recorded so that the+first reader knows what they get.++### 9. Every document is re-chunked on its next reprocess, and nothing is done in advance++`replace_chunks` claims old chunks by exact text. New text for every chunk means every chunk is+superseded and re-inserted, and the document re-extracted. That is the existing reprocess path;+old chunks keep their vectors and their evidence links as before. There is no migration to+re-chunk the whole deployment: the old chunks are not wrong, they are worse, and the next+reprocess of each document is when they improve.++## What the bench found (2026-09-13)++Before, two rounds on `dev` unchanged: **43/52** and **45/52**; relations 12/12 in both; the+stable misses across both rounds were three, none of them present in `extraction_drops` or+`ontology_misses` — never extracted, not extracted and dropped.++After, with `--reprocess` so every chunk is rebuilt. At 1000 tokens: **36/52**, relations 8/12,+the acquisition 8-K 4/11 with seven facts from two chunks — the round that overturned decision 4.+At 300 tokens: **37/52**, but not a measurement: the extraction model's account ran out of+balance (HTTP 402) at 01:22 while the Ohio exhibit was being extracted, and its chunks 3–7 —+the paragraph holding all seven of its numbers — were never sent. On the three documents that+finished before the balance ran out the round scored 29/37, against 29/37 and 33/37 for the two+rounds before; within one standard deviation of both. The round is to be rerun once the account+is funded, and the number that matters is the Ohio exhibit's.++Rerun at 300 tokens with the account funded: **45/52**. The total equals the better baseline round,+but the distribution moved: literals **36/39** (baseline 31 and 32), the Ohio exhibit **15/15**,+relations **8/12** (baseline 12 and 12). Of the four relation misses, three are the Vera Rubin+partners, and they are in the graph — the model wrote `NVIDIA partner CoreWeave` and four more+where the truth reads the platform as the subject; both readings are defensible and this one had+flipped by chance in the baseline rounds. The fourth, Huang's election, was structural, and it is+the first revision to decision 3 above.++Two rounds on the integration of this change with the caption-run revision and the structural+shape checks of #637, extracting the same rebuilt chunks: **43/52**, then **46/52** once the+shape checks read structure instead of words. The second round had relations **12/12** and the+earnings release **19/19**. Its six misses were four Ohio literals, the acquisition's fiscal+year end and one vote count. None of them appears in `extraction_drops`, so the model did not+write them; the Ohio exhibit's literals are the figures that move most between rounds (15/15,+12/15, 11/15). Every round since the change lies within one standard deviation of the baseline,+with no headerless continuation chunk in any of them.++Seen in the rebuilt chunks, independent of the score: no chunk begins inside a table without+its header (was 13 of 35 in the earnings release); the Neal vote table is one table again with+no rule inside it; the chunk holding the non-GAAP EPS carries a header — the parser's layout row,+not the column names, which is the open question below.++## Open questions++- **The budget on its own.** 1000 tokens was chosen to honour the old comment, not measured.+ The same bench at 300 tokens (character parity for English) and at 1000 would separate the+ structural change from the size change. Two runs of twenty minutes each.+- **Evidence that points at a cell or a region.** A fact's evidence is a text span. For a table+ it should be able to name a row and a column; for a figure, a region on a page. That is the+ precondition for extracting from images at all and it is the same ledger discipline —+ document, evidence, fact — extended to two more kinds of evidence. Not this record.+- **The header the parser hands over.** The chunker repeats whatever the parser marked as the+ header. In the XBRL earnings release that is `| | NVIDIA CORPORATION | | … |` — a layout row+ promoted because the table had no `<th>` — while the rows that actually name the columns+ (`Three Months Ended | Six Months Ended`, then the dates) are the first body rows and stay in+ the first piece. Seen on the re-chunked corpus: the chunk holding the non-GAAP EPS now opens+ with a header and still does not say which quarter is which. The mechanism is right and the+ input is wrong; choosing the header rows is `promote_first_row_headers`' job and belongs with+ the `head_blank` rule already there. Not this cut.+- **DOCX and PDF tables.** The DOCX parser flattens tables to paragraphs; the PDF parser has no+ tables. Both arrive at the block reader as prose and get prose treatment. Spreadsheets and CSV+ come out as tab- and pipe-separated lines rather than Markdown tables and are likewise read as+ paragraphs. Emitting real Markdown tables from those four parsers is cheap for the first two+ and is cut 2 for the last two.diff --git a/docs/decisions/README.md b/docs/decisions/README.mdindex b92bdbae..1b75ab03 100644--- a/docs/decisions/README.md+++ b/docs/decisions/README.md@@ -62,6 +62,7 @@ The test for writing one: if someone (including us) looks at a piece of code in| 0036 | [Exploration aligns a schema to the ontology](0036-exploration-aligns-a-schema-to-the-ontology.md) | Written, not implemented · exploration proposes an alignment per table (the class it is a table of, its attributes, its relations, a conversion tree per column) through `ontology_proposals`, adopted as one thing · a definition is a rule over aligned attributes, written by a person; a shared convention is one rule others read · conversions are 0032's expression tree, text parsed by `sqlparser`, never stored as SQL · `Metric` / `Dimension` retire and `concept_mappings` becomes rendered · the schema document leaves extraction · where a rule runs against the source stays open || 0037 | [A relation carries its own attributes](0037-a-relation-carries-its-own-attributes.md) | Written · cut 1 in progress: qualifier tables (0049), declaration, extraction, panel, export · entity-valued qualifier reserved, conflict-as-two-rows and the canvas label next || 0038 | [The interface has a light side](0038-the-interface-has-a-light-side.md) | Implemented (#599) · `data-theme` on `<html>`, chosen by the reader and stored in the browser, `system` resolved to one of the two before first paint · a second token block in `styles.css` and not one rule rewritten: alpha stays, the triplet swaps · the canvas reads its colours from the tokens through one reader file and rebuilds on a switch; edges are flattened over the ground on light because the WebGL shader can only brighten · a `raw-colour` guard rule over `.ts` and `.tsx` · the entity palette is the same in both themes; a paper-tuned set is open |+| 0039 | [A chunk is what extraction sees](0039-a-chunk-is-what-extraction-sees.md) | Implemented, cut 1 · the parser's Markdown is read into blocks and packed by a token budget (1000, cl100k) · a table travels with its caption and header and a continuation repeats both · headings become a breadcrumb and fill `chunks.heading` · a page-break rule is not a boundary and a table it split is joined back · not measured on its own: the budget · cut 2: an external parser behind the block model, evidence that names a cell or a region |## Not a decision recorddiff --git a/scripts/bench/recall.mjs b/scripts/bench/recall.mjsindex 7cffe2a8..faf901ad 100644--- a/scripts/bench/recall.mjs+++ b/scripts/bench/recall.mjs@@ -202,7 +202,8 @@ psql(`DELETE FROM extraction_drops WHERE kb_id='${KB}'`);psql(`DELETE FROM ontology_misses WHERE kb_id='${KB}'`);// 自动扩本体上一轮长出来的关系:留着就进下一轮的提示词,两轮条件不同psql(`DELETE FROM relation_types WHERE kb_id='${KB}' AND created_at > '${packTs}'::timestamptz + interval '1 second'`);-psql(`UPDATE knowledge_bases SET auto_extend_ontology=FALSE, auto_type_resolution=FALSE WHERE id='${KB}'`);+// 四个会在抽取之后改图的开关一律关掉:测的是抽取本身,推理物化和治理闸门会在打分前增删事实+psql(`UPDATE knowledge_bases SET auto_extend_ontology=FALSE, auto_type_resolution=FALSE, materialize_inferences=FALSE, governance=FALSE WHERE id='${KB}'`);psql(`UPDATE chunks SET extracted_at=NULL WHERE document_id IN (SELECT id FROM documents WHERE kb_id='${KB}' AND filename IN (${names}))`);console.log(`本体 ${num(`SELECT count(*) FROM relation_types WHERE kb_id='${KB}'`)} 个关系 / ${num(`SELECT count(*) FROM entity_types WHERE kb_id='${KB}'`)} 个类`);