Skip to content

World's first open-source enterprise world model.

A chunk keeps its table header (#633)

WaylandYang45953d1ea170
diff --git a/Cargo.lock b/Cargo.lock
index 2a464102..9642ace7 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -475,6 +475,17 @@ dependencies = [
"syn 3.0.5",
]
+[[package]]
+name = "bstr"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f"
+dependencies = [
+ "memchr",
+ "regex-automata",
+ "serde_core",
+]
+
[[package]]
name = "bumpalo"
version = "3.20.3"
@@ -1310,7 +1321,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -1343,6 +1354,17 @@ dependencies = [
"pin-project-lite",
]
+[[package]]
+name = "fancy-regex"
+version = "0.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8"
+dependencies = [
+ "bit-set",
+ "regex-automata",
+ "regex-syntax",
+]
+
[[package]]
name = "fast-float2"
version = "0.2.4"
@@ -2687,7 +2709,7 @@ version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -2880,7 +2902,7 @@ dependencies = [
"oxiri",
"oxrdf",
"ryu-js",
- "thiserror 1.0.69",
+ "thiserror 2.0.20",
]
[[package]]
@@ -2892,7 +2914,7 @@ dependencies = [
"oxilangtag",
"oxiri",
"rand 0.9.5",
- "thiserror 1.0.69",
+ "thiserror 2.0.20",
]
[[package]]
@@ -2905,7 +2927,7 @@ dependencies = [
"oxrdf",
"oxrdfxml",
"oxttl",
- "thiserror 1.0.69",
+ "thiserror 2.0.20",
]
[[package]]
@@ -2918,7 +2940,7 @@ dependencies = [
"oxiri",
"oxrdf",
"quick-xml 0.37.5",
- "thiserror 1.0.69",
+ "thiserror 2.0.20",
]
[[package]]
@@ -2931,7 +2953,7 @@ dependencies = [
"oxilangtag",
"oxiri",
"oxrdf",
- "thiserror 1.0.69",
+ "thiserror 2.0.20",
]
[[package]]
@@ -3293,6 +3315,17 @@ dependencies = [
"cc",
]
+[[package]]
+name = "pulldown-cmark"
+version = "0.13.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e9f068eba8e7071c5f9511831b44f32c740d5adf574e990f946ddb53db2f314e"
+dependencies = [
+ "bitflags",
+ "memchr",
+ "unicase",
+]
+
[[package]]
name = "quick-xml"
version = "0.37.5"
@@ -3376,7 +3409,7 @@ dependencies = [
"once_cell",
"socket2",
"tracing",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -3707,7 +3740,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -3765,7 +3798,7 @@ dependencies = [
"security-framework",
"security-framework-sys",
"webpki-root-certs",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -4666,10 +4699,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
dependencies = [
"fastrand",
- "getrandom 0.3.4",
+ "getrandom 0.4.3",
"once_cell",
"rustix",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -4696,6 +4729,7 @@ dependencies = [
"memchr",
"strum 0.28.0",
"thiserror 2.0.20",
+ "tiktoken-rs",
]
[[package]]
@@ -4747,6 +4781,21 @@ dependencies = [
"cfg-if",
]
+[[package]]
+name = "tiktoken-rs"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "027853bbf8c7763b77c5c595f1c271c7d536ced7d6f83452911b944621e57fc2"
+dependencies = [
+ "anyhow",
+ "base64 0.22.1",
+ "bstr",
+ "fancy-regex",
+ "lazy_static",
+ "regex",
+ "rustc-hash",
+]
+
[[package]]
name = "time"
version = "0.3.55"
@@ -5185,9 +5234,11 @@ dependencies = [
"oxttl",
"pdf-extract",
"percent-encoding",
+ "pulldown-cmark",
"quick-xml 0.42.0",
"text-splitter",
"thiserror 2.0.20",
+ "tiktoken-rs",
"tracing",
"zip 6.0.0",
]
@@ -5529,7 +5580,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
- "windows-sys 0.48.0",
+ "windows-sys 0.61.2",
]
[[package]]
diff --git a/Cargo.toml b/Cargo.toml
index 351d61cf..5afaa522 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -60,7 +60,9 @@ utopia-search = { path = "crates/utopia-search" }
utopia-llm = { path = "crates/utopia-llm" }
tantivy = "0.26"
tantivy-jieba = "0.20"
-text-splitter = "0.32"
+text-splitter = { version = "0.32", features = ["tiktoken-rs"] }
+# 分块预算按 token 数:cl100k 的排名表随 crate 内嵌,离线可用
+tiktoken-rs = "0.12"
pdf-extract = "0.12"
calamine = "0.36"
pgvector = { version = "0.4", features = ["sqlx"] }
diff --git a/crates/utopia-core/src/config.rs b/crates/utopia-core/src/config.rs
index 7ed40ac6..e1af1683 100644
--- a/crates/utopia-core/src/config.rs
+++ b/crates/utopia-core/src/config.rs
@@ -26,6 +26,9 @@ pub struct AppConfig {
pub web_dist: String,
/// 数据目录:原始文件(files/)与 Tantivy 索引(index/)。
pub data_dir: String,
+ /// 一个分块的预算(cl100k token)。是个旋钮而不是常量,因为它要能被量:
+ /// 结构(表头跟着走)与大小是两件事,召回台子得能只动一个。`UTOPIA_CHUNK_TOKENS`
+ pub chunk_tokens: usize,
/// 数据库连接池上限。缺省 32,与 worker 并发的缺省对齐——池子小于并发时
/// 症状是请求变慢而不是任何一处说"池子不够",所以它必须可调。
pub db_max_connections: Option<u32>,
@@ -46,6 +49,7 @@ impl Default for AppConfig {
secret_key: None,
web_dist: "web/dist".into(),
data_dir: "data".into(),
+ chunk_tokens: 300,
db_max_connections: None,
cookie_secure: false,
open_registration: true,
@@ -155,6 +159,7 @@ mod tests {
jail.set_env("UTOPIA_BIND_ADDR", "");
let cfg = AppConfig::load().unwrap();
assert_eq!(cfg.data_dir, "data");
+ assert_eq!(cfg.chunk_tokens, 300);
assert_eq!(cfg.web_dist, "web/dist");
assert_eq!(cfg.bind_addr, "0.0.0.0:1516");
Ok(())
diff --git a/crates/utopia-ingest/Cargo.toml b/crates/utopia-ingest/Cargo.toml
index b8dee99c..7b620f68 100644
--- a/crates/utopia-ingest/Cargo.toml
+++ b/crates/utopia-ingest/Cargo.toml
@@ -7,6 +7,8 @@ license.workspace = true
[dependencies]
anyhow.workspace = true
text-splitter.workspace = true
+tiktoken-rs.workspace = true
+pulldown-cmark.workspace = true
pdf-extract.workspace = true
calamine.workspace = true
htmd.workspace = true
diff --git a/crates/utopia-ingest/src/blocks.rs b/crates/utopia-ingest/src/blocks.rs
new file mode 100644
index 00000000..ee9fafba
--- /dev/null
+++ b/crates/utopia-ingest/src/blocks.rs
@@ -0,0 +1,297 @@
+//! 文档块:把解析器出的 Markdown 读成**顶层块**的序列,每块带着自己在原文里的
+//! 字节范围。分块器在这一层之上打包,而不是直接对着字符串找边界。
+//!
+//! 为什么要这一层:字符串里没有「这是一张表的第三行」这个信息,切分器只能靠
+//! 换行和空行猜。猜出来的结果量过(召回台子,2026-09-13):财报 35 块里 13 块
+//! 从表格中间开始、没有表头,模型面对一行五个美元数只能猜哪列是哪季度;投票
+//! 结果里一张表被原文的分页符劈成两半,后半截成了没有名字的孤表。
+//!
+//! 这里只做两件事:读出块,和把被分页符劈开的表接回去。切多大、怎么补头,
+//! 归 `chunker`。
+
+use pulldown_cmark::{Event, Options, Parser, Tag, TagEnd};
+use std::ops::Range;
+
+/// 一个顶层块是什么。范围一律对齐到整行,切出来的文本一个字不改。
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum Kind {
+ /// `#` 的个数
+ Heading(u8),
+ Paragraph,
+ /// `head` 是表头行连同分隔行;`rows` 是体行,一行一个范围。
+ /// 接回来的续表让 `head`/`rows` 不再连续,所以表的文本一律由这两样拼,
+ /// 不从 `Block::range` 整段切
+ Table {
+ head: Range<usize>,
+ rows: Vec<Range<usize>>,
+ },
+ /// 水平线。在这类文档里多半是**分页符**,不是章节的边界
+ Rule,
+ /// 列表、代码块、引用、原样 HTML:整块一个单元,不往里看
+ Other,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Block {
+ pub kind: Kind,
+ pub range: Range<usize>,
+}
+
+/// 读出顶层块,并把被分页符劈开的表接回去。
+pub fn blocks(text: &str) -> Vec<Block> {
+ join_continuations(text, parse(text))
+}
+
+/// 正在收的表:整表范围、表头范围、体行范围
+type OpenTable = (Range<usize>, Option<Range<usize>>, Vec<Range<usize>>);
+
+fn parse(text: &str) -> Vec<Block> {
+ let mut out: Vec<Block> = Vec::new();
+ let mut depth = 0usize;
+ let mut table: Option<OpenTable> = None;
+ for (event, range) in Parser::new_ext(text, Options::ENABLE_TABLES).into_offset_iter() {
+ match event {
+ Event::Start(tag) => {
+ if depth == 0 {
+ let kind = match &tag {
+ Tag::Heading { level, .. } => Some(Kind::Heading(*level as u8)),
+ Tag::Paragraph => Some(Kind::Paragraph),
+ Tag::Table(_) => {
+ table = Some((range.clone(), None, Vec::new()));
+ None
+ }
+ _ => Some(Kind::Other),
+ };
+ if let Some(kind) = kind {
+ out.push(Block {
+ kind,
+ range: lines(text, range),
+ });
+ }
+ } else if depth == 1 {
+ if let Some((_, head, rows)) = table.as_mut() {
+ match tag {
+ Tag::TableHead => *head = Some(lines(text, range)),
+ Tag::TableRow => rows.push(lines(text, range)),
+ _ => {}
+ }
+ }
+ }
+ depth += 1;
+ }
+ Event::End(tag) => {
+ depth = depth.saturating_sub(1);
+ if depth == 0 && tag == TagEnd::Table {
+ if let Some((whole, head, rows)) = table.take() {
+ let whole = lines(text, whole);
+ // 表头连同它下面的分隔行:从表的开头到第一个体行之前
+ let head = match (head, rows.first()) {
+ (Some(_), Some(first)) => whole.start..line_before(text, first.start),
+ _ => whole.clone(),
+ };
+ out.push(Block {
+ kind: Kind::Table { head, rows },
+ range: whole,
+ });
+ }
+ }
+ }
+ Event::Rule if depth == 0 => out.push(Block {
+ kind: Kind::Rule,
+ range: lines(text, range),
+ }),
+ _ => {}
+ }
+ }
+ out
+}
+
+/// 被分页符劈开的表接回去。
+///
+/// 形状很具体:`表 A · 分隔线 · 表 B`,两张一样宽,而 B 的「表头」首格跟 A 某个
+/// 体行的首格共享开头两个词——`Number of shares Abstaining` 接在
+/// `Number of shares For / Against` 后面。那不是 B 的表头,是 A 的下一行,
+/// 解析器只是因为分隔行在它上面才把它当成了表头。
+///
+/// 判据故意窄:只看行首的标签,不看数。「同宽、表头里有数字」会把两张按年份
+/// 分列的财务表接成一张——`2025 | 2024` 也是数字。
+fn join_continuations(text: &str, blocks: Vec<Block>) -> Vec<Block> {
+ let mut out: Vec<Block> = Vec::with_capacity(blocks.len());
+ let mut i = 0usize;
+ while i < blocks.len() {
+ let b = blocks[i].clone();
+ let joined = match (&b.kind, out.last()) {
+ (
+ Kind::Rule,
+ Some(Block {
+ kind: Kind::Table { .. },
+ ..
+ }),
+ ) => match blocks.get(i + 1) {
+ Some(Block {
+ kind: Kind::Table { head, rows },
+ range,
+ }) if continues(text, out.last().unwrap(), head) => {
+ let Some(Block {
+ kind:
+ Kind::Table {
+ rows: prev_rows, ..
+ },
+ range: prev_range,
+ }) = out.last_mut()
+ else {
+ unreachable!()
+ };
+ // B 的「表头行」是 A 的一个体行;分隔行不要
+ prev_rows.push(first_line(text, head));
+ prev_rows.extend(rows.iter().cloned());
+ prev_range.end = range.end;
+ i += 2;
+ true
+ }
+ _ => false,
+ },
+ _ => false,
+ };
+ if !joined {
+ out.push(b);
+ i += 1;
+ }
+ }
+ out
+}
+
+fn continues(text: &str, prev: &Block, next_head: &Range<usize>) -> bool {
+ let Kind::Table { head, rows } = &prev.kind else {
+ return false;
+ };
+ let width = |r: &Range<usize>| cell_count(&text[first_line(text, r)]);
+ if width(head) != width(next_head) {
+ return false;
+ }
+ let lead = |r: &Range<usize>| {
+ first_cell(&text[first_line(text, r)])
+ .split_whitespace()
+ .take(2)
+ .map(str::to_lowercase)
+ .collect::<Vec<_>>()
+ };
+ let next = lead(next_head);
+ next.len() == 2 && rows.iter().any(|r| lead(r) == next)
+}
+
+/// 范围对齐到整行:往前退到行首,往后走到行尾(不含换行)。
+fn lines(text: &str, r: Range<usize>) -> Range<usize> {
+ let start = text[..r.start.min(text.len())]
+ .rfind('\n')
+ .map_or(0, |i| i + 1);
+ let mut end = r.end.min(text.len());
+ while end > start && matches!(text.as_bytes()[end - 1], b'\n' | b' ' | b'\t') {
+ end -= 1;
+ }
+ let mut end = text[end..].find('\n').map_or(text.len(), |i| end + i);
+ // 扩到行尾会把行尾的空格再收回来;块的文本不带尾随空白
+ while end > start && matches!(text.as_bytes()[end - 1], b' ' | b'\t') {
+ end -= 1;
+ }
+ start..end
+}
+
+/// 上一行的末尾(不含换行):一个范围结束在哪儿,下一段从哪儿起
+fn line_before(text: &str, at: usize) -> usize {
+ let mut end = at;
+ while end > 0 && matches!(text.as_bytes()[end - 1], b'\n' | b' ' | b'\t') {
+ end -= 1;
+ }
+ end
+}
+
+fn first_line(text: &str, r: &Range<usize>) -> Range<usize> {
+ let end = text[r.start..r.end]
+ .find('\n')
+ .map_or(r.end, |i| r.start + i);
+ r.start..end
+}
+
+fn cell_count(line: &str) -> usize {
+ let t = line.trim();
+ if t.starts_with('|') && t.ends_with('|') && t.len() > 1 {
+ t[1..t.len() - 1].split('|').count()
+ } else {
+ 0
+ }
+}
+
+fn first_cell(line: &str) -> &str {
+ let t = line.trim();
+ t.trim_start_matches('|')
+ .split('|')
+ .next()
+ .unwrap_or("")
+ .trim()
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn kinds(text: &str) -> Vec<String> {
+ blocks(text)
+ .into_iter()
+ .map(|b| match b.kind {
+ Kind::Heading(l) => format!("h{l}"),
+ Kind::Paragraph => "p".into(),
+ Kind::Table { rows, .. } => format!("table({})", rows.len()),
+ Kind::Rule => "rule".into(),
+ Kind::Other => "other".into(),
+ })
+ .collect()
+ }
+
+ #[test]
+ fn a_table_keeps_its_header_and_rows_apart() {
+ let text = "Intro.\n\n| a | b |\n| --- | --- |\n| 1 | 2 |\n| 3 | 4 |\n\nAfter.";
+ let b = blocks(text);
+ assert_eq!(kinds(text), ["p", "table(2)", "p"]);
+ let Kind::Table { head, rows } = &b[1].kind else {
+ panic!()
+ };
+ assert_eq!(&text[head.clone()], "| a | b |\n| --- | --- |");
+ assert_eq!(&text[rows[0].clone()], "| 1 | 2 |");
+ assert_eq!(&text[rows[1].clone()], "| 3 | 4 |");
+ assert_eq!(
+ &text[b[1].range.clone()],
+ "| a | b |\n| --- | --- |\n| 1 | 2 |\n| 3 | 4 |"
+ );
+ }
+
+ /// 投票结果 8-K 里 Stephen C. Neal 那张表的形状:原文的分页符落在表中间,
+ /// 后半截带着一条分隔行,解析器把「弃权」那一行当成了新表的表头。
+ #[test]
+ fn a_headerless_table_after_a_rule_continues_the_one_before() {
+ let text = "| g. Stephen C. Neal | |\n| --- | --- |\n| Number of shares For | 14,573 |\n| Number of shares Against | 2,234 |\n\n* * *\n\n| Number of shares Abstaining | 49,064 |\n| --- | --- |\n| Number of Broker Non-Votes | 2,829 |\n";
+ assert_eq!(kinds(text), ["table(4)"]);
+ let Kind::Table { rows, .. } = &blocks(text)[0].kind else {
+ panic!()
+ };
+ let rows: Vec<&str> = rows.iter().map(|r| &text[r.clone()]).collect();
+ assert_eq!(rows[2], "| Number of shares Abstaining | 49,064 |");
+ assert_eq!(rows[3], "| Number of Broker Non-Votes | 2,829 |");
+ }
+
+ /// 两张按年份分列的表隔着分页符:表头是 `2025 | 2024` 这种数字,不接。
+ #[test]
+ fn two_tables_with_numeric_headers_stay_two_tables() {
+ let text = "| Revenue | 2025 |\n| --- | --- |\n| Total | 10 |\n\n* * *\n\n| 2024 | 2023 |\n| --- | --- |\n| Total | 9 |\n";
+ assert_eq!(kinds(text), ["table(1)", "rule", "table(1)"]);
+ }
+
+ #[test]
+ fn headings_rules_and_lists_are_their_own_blocks() {
+ let text = "# Title\n\nPara one.\n\n- a\n- b\n\n---\n\n## Sub\n\nPara two.";
+ assert_eq!(kinds(text), ["h1", "p", "other", "rule", "h2", "p"]);
+ let b = blocks(text);
+ assert_eq!(&text[b[0].range.clone()], "# Title");
+ assert_eq!(&text[b[2].range.clone()], "- a\n- b");
+ }
+}
diff --git a/crates/utopia-ingest/src/chunker.rs b/crates/utopia-ingest/src/chunker.rs
index 09575035..405bad8c 100644
--- a/crates/utopia-ingest/src/chunker.rs
+++ b/crates/utopia-ingest/src/chunker.rs
@@ -1,6 +1,26 @@
-//! 分块:text-splitter 语义分块(段落/句子边界优先),字符预算 + 重叠。
+//! 分块:把文档块打包成抽取一次能看见的单元。
+//!
+//! **块的职责是「抽取一次能看见的全部」。**模型只看这一块,切开的东西补不回来,
+//! 所以块边界是抽取契约的一部分,不是文本处理的细节。从这条出发定的规矩:
+//!
+//! - 一行不拆;表头与表体不拆;说明句(表上面那句「结果如下:」)与它的表不拆;
+//! 标题与它下面的第一段不拆。
+//! - 一张表放得下就整张放。放不下才按行切,**每个续块重复面包屑、说明句和表头**
+//! ——这一条才是修「续块只剩一行五个美元数」的那一刀,换再聪明的切分器都不解决它。
+//! - 预算按 token 算,不按字符。从前是 1200 字符,注释说「约 1000+ token」,那是中文;
+//! 英文只有 300 上下,一张宽表放不下。现在两种语言拿到的是同一个预算。
+//! - 水平线**不是**切点。从前的切分器偏爱在水平线上切,而这类文档里水平线是分页符:
+//! 投票结果里一张表正是被它劈成了两半。
+//! - 不重叠。块是语义单元,上下文靠面包屑和补头给,不靠把上一块的尾巴抄一遍。
+//!
+//! 每块的正文都是原文的切片,一个字不改;前缀(面包屑、说明句、表头)也是原文的
+//! 切片,只是从别处抄来的。`char_start` / `char_end` 指正文那一段,不含前缀。
+use crate::blocks::{blocks, Block, Kind};
+use std::ops::Range;
+use std::sync::OnceLock;
use text_splitter::{ChunkConfig, TextSplitter};
+use tiktoken_rs::CoreBPE;
#[derive(Debug, Clone)]
pub struct ChunkPiece {
@@ -8,22 +28,691 @@ pub struct ChunkPiece {
pub text: String,
pub char_start: i32,
pub char_end: i32,
+ /// 这块所在的章节路径,`›` 分隔;没有标题的文档为 None
+ pub heading: Option<String>,
}
-/// 默认预算:1200 字符(中文约等于 1000+ token),重叠 150。
+/// 一块的预算(cl100k token)。
+///
+/// **300,不是 1000。**1000 是旧字符预算的注释里写的本意,第一轮就量掉了:收购
+/// 8-K 从九块变成两块,模型对着 4700 字符只写了七条事实——地址、电话、「published
+/// in the SEC」——收购本身一条都没有。模型一次调用写出的事实数不随输入变长而变多,
+/// 块一大它就挑最容易的几条写。300 是旧切法在英文上的实际大小,也是 43/45 那两轮
+/// 量出来的条件;中文在这个数下拿到的上下文比从前少,还没量(0039 开放问题)。
+pub const BUDGET_TOKENS: usize = 300;
+
+/// 说明句最长多少字节还算说明句:表上面那一段要短、或者以冒号结尾,
+/// 才当成表的一部分带着走;一整段分析不是说明句
+const CAPTION_MAX_BYTES: usize = 200;
+
+/// 不到这么多 token 的一段(页码、脚注标记)不单独成块,并进上一块
+const TINY_TOKENS: usize = 8;
+
pub fn chunk_text(text: &str) -> Vec<ChunkPiece> {
- let config = ChunkConfig::new(1200)
- .with_overlap(150)
- .expect("overlap < capacity");
- let splitter = TextSplitter::new(config);
- splitter
- .chunk_indices(text)
+ chunk_with_budget(text, BUDGET_TOKENS)
+}
+
+fn bpe() -> &'static CoreBPE {
+ static BPE: OnceLock<CoreBPE> = OnceLock::new();
+ BPE.get_or_init(|| tiktoken_rs::cl100k_base().expect("cl100k 的排名表随 crate 内嵌"))
+}
+
+fn tokens(s: &str) -> usize {
+ bpe().encode_ordinary(s).len()
+}
+
+/// 打包的单元:标题贴着下一块走;表带着自己的说明句。
+enum Unit {
+ Heading {
+ level: u8,
+ range: Range<usize>,
+ },
+ Text(Range<usize>),
+ Table {
+ caption: Option<Range<usize>>,
+ head: Range<usize>,
+ rows: Vec<Range<usize>>,
+ },
+}
+
+fn units(text: &str, blocks: Vec<Block>) -> Vec<Unit> {
+ let mut out: Vec<Unit> = Vec::with_capacity(blocks.len());
+ let mut i = 0usize;
+ while i < blocks.len() {
+ let b = &blocks[i];
+ match &b.kind {
+ Kind::Heading(level) => out.push(Unit::Heading {
+ level: *level,
+ range: b.range.clone(),
+ }),
+ Kind::Rule => {}
+ Kind::Paragraph => {
+ // 紧跟着表的短段落是说明句,跟表走。**两条通用的补充**(召回台子,
+ // 2026-09-13 充值后那一轮量出来的):
+ //
+ // - 说明句和表之间隔着水平线也算紧跟。这类文档里水平线是分页符,
+ // 第五项议案的「结果如下:」和它的表正好被一页隔开,说明句留在了
+ // 上一块,表去了下一块。
+ // - 一句说明引出的是**紧跟其后的一串表**,不只第一张。「选举十位董事
+ // 提名人,结果如下:」后面是十张表,从前只有第一张带着它;黄仁勋
+ // 那张在另一块里,四个票数都对,却没有一句话说这是在投什么,
+ // 「当选董事」那条边就没了。一串到别的东西(段落、标题)出现为止。
+ let caption_like = b.range.len() <= CAPTION_MAX_BYTES
+ || text[b.range.clone()].trim_end().ends_with(':');
+ let mut j = i + 1;
+ let mut attached = 0usize;
+ if caption_like {
+ loop {
+ while matches!(
+ blocks.get(j),
+ Some(Block {
+ kind: Kind::Rule,
+ ..
+ })
+ ) {
+ j += 1;
+ }
+ let Some(Block {
+ kind: Kind::Table { head, rows },
+ ..
+ }) = blocks.get(j)
+ else {
+ break;
+ };
+ out.push(Unit::Table {
+ caption: Some(b.range.clone()),
+ head: head.clone(),
+ rows: rows.clone(),
+ });
+ attached += 1;
+ j += 1;
+ }
+ }
+ if attached == 0 {
+ out.push(Unit::Text(b.range.clone()));
+ } else {
+ i = j - 1;
+ }
+ }
+ Kind::Table { head, rows } => out.push(Unit::Table {
+ caption: None,
+ head: head.clone(),
+ rows: rows.clone(),
+ }),
+ Kind::Other => out.push(Unit::Text(b.range.clone())),
+ }
+ i += 1;
+ }
+ out
+}
+
+/// 正文里的一项:一段文本,或一张(可能只剩几行的)表
+enum Piece {
+ Text(Range<usize>),
+ Table {
+ caption: Option<Range<usize>>,
+ head: Range<usize>,
+ rows: Vec<Range<usize>>,
+ },
+}
+
+impl Piece {
+ fn render(&self, text: &str) -> String {
+ match self {
+ Piece::Text(r) => text[r.clone()].to_string(),
+ Piece::Table {
+ caption,
+ head,
+ rows,
+ } => {
+ let mut s = String::new();
+ if let Some(c) = caption {
+ s.push_str(&text[c.clone()]);
+ s.push_str("\n\n");
+ }
+ s.push_str(&text[head.clone()]);
+ for r in rows {
+ s.push('\n');
+ s.push_str(&text[r.clone()]);
+ }
+ s
+ }
+ }
+ }
+ fn span(&self) -> Range<usize> {
+ match self {
+ Piece::Text(r) => r.clone(),
+ Piece::Table {
+ caption,
+ head,
+ rows,
+ } => {
+ let start = caption.as_ref().map_or(head.start, |c| c.start);
+ let end = rows.last().map_or(head.end, |r| r.end);
+ start..end
+ }
+ }
+ }
+}
+
+struct Packer<'a> {
+ text: &'a str,
+ budget: usize,
+ out: Vec<ChunkPiece>,
+ /// 现在在哪一章:(层级, 标题行)
+ path: Vec<(u8, Range<usize>)>,
+ /// 出现了、还没贴到正文上的标题
+ pending: Vec<Range<usize>>,
+ /// 正在攒的块:前缀(章节标题)与正文
+ prefix: Vec<Range<usize>>,
+ body: Vec<Piece>,
+}
+
+impl<'a> Packer<'a> {
+ fn render(&self, prefix: &[Range<usize>], body: &[Piece]) -> String {
+ let mut parts: Vec<String> = prefix
+ .iter()
+ .map(|r| self.text[r.clone()].to_string())
+ .collect();
+ // 同一块里一串表共用一句说明:说明只在第一张前面出现一次。
+ // 十位董事的十张表挤在一块里,不该把「结果如下」念十遍
+ let mut last_caption: Option<Range<usize>> = None;
+ for p in body {
+ match p {
+ Piece::Table {
+ caption: Some(c),
+ head,
+ rows,
+ } if last_caption.as_ref() == Some(c) => {
+ let bare = Piece::Table {
+ caption: None,
+ head: head.clone(),
+ rows: rows.clone(),
+ };
+ parts.push(bare.render(self.text));
+ }
+ Piece::Table { caption, .. } => {
+ last_caption = caption.clone();
+ parts.push(p.render(self.text));
+ }
+ Piece::Text(_) => {
+ last_caption = None;
+ parts.push(p.render(self.text));
+ }
+ }
+ }
+ parts.join("\n\n")
+ }
+
+ /// 新块的前缀:现在这一章的标题路径,去掉马上要作为正文出现的那几条
+ fn prefix_now(&self) -> Vec<Range<usize>> {
+ self.path
+ .iter()
+ .map(|(_, r)| r.clone())
+ .filter(|r| !self.pending.contains(r))
+ .collect()
+ }
+
+ fn fits(&self, prefix: &[Range<usize>], body: &[Piece]) -> bool {
+ tokens(&self.render(prefix, body)) <= self.budget
+ }
+
+ /// 试着把一项放进当前块;放不下就先收当前块,再试一次空块。
+ ///
+ /// **极小的一项不单独成块。**新闻稿末尾有一行页码「4」,按预算它放不进已经
+ /// 满了的上一块,于是自己成了一块——一个字符的块,抽取照样为它调一次模型。
+ /// 几个 token 的东西并进上一块,超预算那几个 token 无所谓。
+ fn place(&mut self, piece: Piece) -> Option<Piece> {
+ if self.pending.is_empty() {
+ if let Piece::Text(r) = &piece {
+ if tokens(&self.text[r.clone()]) <= TINY_TOKENS {
+ if !self.body.is_empty() {
+ self.body.push(piece);
+ return None;
+ }
+ // 上一块已经发出去了(长段落走退路切分会立刻 flush):接到它尾上
+ if let Some(last) = self.out.last_mut() {
+ last.text.push_str(
+ "
+
+",
+ );
+ last.text.push_str(&self.text[r.clone()]);
+ last.char_end = last.char_end.max(r.end as i32);
+ return None;
+ }
+ }
+ }
+ }
+ let mut with_headings: Vec<Piece> = self.pending.iter().cloned().map(Piece::Text).collect();
+ with_headings.push(piece);
+ if self.body.is_empty() {
+ self.prefix = self.prefix_now();
+ }
+ let mut candidate: Vec<Piece> = std::mem::take(&mut self.body);
+ candidate.extend(with_headings);
+ if self.fits(&self.prefix.clone(), &candidate) {
+ self.body = candidate;
+ self.pending.clear();
+ return None;
+ }
+ // 放不下:把刚加的拆回来
+ let n = candidate.len() - (self.pending.len() + 1);
+ let mut rest = candidate.split_off(n);
+ self.body = candidate;
+ let piece = rest.pop().expect("刚放进去的那一项");
+ if !self.body.is_empty() {
+ self.flush();
+ return self.place(piece);
+ }
+ Some(piece)
+ }
+
+ fn flush(&mut self) {
+ if self.body.is_empty() {
+ return;
+ }
+ let body = std::mem::take(&mut self.body);
+ let prefix = std::mem::take(&mut self.prefix);
+ let text = self.render(&prefix, &body);
+ let start = body.iter().map(|p| p.span().start).min().unwrap_or(0);
+ let end = body.iter().map(|p| p.span().end).max().unwrap_or(0);
+ let heading = self.breadcrumb();
+ self.out.push(ChunkPiece {
+ seq: self.out.len() as i32,
+ text,
+ char_start: start as i32,
+ char_end: end as i32,
+ heading,
+ });
+ }
+
+ fn breadcrumb(&self) -> Option<String> {
+ if self.path.is_empty() {
+ return None;
+ }
+ Some(
+ self.path
+ .iter()
+ .map(|(_, r)| self.text[r.clone()].trim_start_matches('#').trim())
+ .collect::<Vec<_>>()
+ .join(" › "),
+ )
+ }
+
+ fn heading(&mut self, level: u8, range: Range<usize>) {
+ while self.path.last().is_some_and(|(l, _)| *l >= level) {
+ self.path.pop();
+ }
+ self.path.push((level, range.clone()));
+ self.pending.push(range);
+ }
+
+ fn text_unit(&mut self, range: Range<usize>) {
+ let Some(Piece::Text(range)) = self.place(Piece::Text(range)) else {
+ return;
+ };
+ // 一段就超预算:退回按句切,每一小段自成一块,前缀照给
+ self.prefix = self.prefix_now();
+ let prefix_cost = tokens(&self.render(&self.prefix.clone(), &[]));
+ let headings: Vec<Piece> = self.pending.drain(..).map(Piece::Text).collect();
+ let heading_cost = tokens(&self.render(&[], &headings));
+ let capacity = self
+ .budget
+ .saturating_sub(prefix_cost + heading_cost)
+ .max(1);
+ let config = ChunkConfig::new(capacity).with_sizer(bpe());
+ let splitter = TextSplitter::new(config);
+ let block = &self.text[range.clone()];
+ let mut first = true;
+ for (offset, piece) in splitter.chunk_indices(block) {
+ let sub = range.start + offset..range.start + offset + piece.len();
+ if first {
+ self.body = headings
+ .iter()
+ .map(|h| match h {
+ Piece::Text(r) => Piece::Text(r.clone()),
+ _ => unreachable!(),
+ })
+ .collect();
+ first = false;
+ }
+ self.body.push(Piece::Text(sub));
+ self.prefix = self.prefix_now();
+ self.flush();
+ }
+ }
+
+ fn table_unit(
+ &mut self,
+ caption: Option<Range<usize>>,
+ head: Range<usize>,
+ rows: Vec<Range<usize>>,
+ ) {
+ let whole = Piece::Table {
+ caption: caption.clone(),
+ head: head.clone(),
+ rows: rows.clone(),
+ };
+ let Some(_) = self.place(whole) else {
+ return;
+ };
+ // 整张放不下:按行切,每一块都带说明句和表头。一行不拆;
+ // 说明句 + 表头 + 一行还超预算的,照样出一块——没有更好的切法
+ let mut rows = rows.into_iter().peekable();
+ while rows.peek().is_some() {
+ let mut taken: Vec<Range<usize>> = vec![rows.next().expect("peek 过")];
+ while let Some(next) = rows.peek() {
+ let mut trial = taken.clone();
+ trial.push(next.clone());
+ let piece = Piece::Table {
+ caption: caption.clone(),
+ head: head.clone(),
+ rows: trial,
+ };
+ let headings: Vec<Piece> = self.pending.iter().cloned().map(Piece::Text).collect();
+ let mut body = headings;
+ body.push(piece);
+ if self.fits(&self.prefix_now(), &body) {
+ taken.push(rows.next().expect("peek 过"));
+ } else {
+ break;
+ }
+ }
+ self.prefix = self.prefix_now();
+ self.body = self.pending.drain(..).map(Piece::Text).collect();
+ self.body.push(Piece::Table {
+ caption: caption.clone(),
+ head: head.clone(),
+ rows: taken,
+ });
+ self.flush();
+ }
+ }
+}
+
+pub fn chunk_with_budget(text: &str, budget: usize) -> Vec<ChunkPiece> {
+ let mut p = Packer {
+ text,
+ budget,
+ out: Vec::new(),
+ path: Vec::new(),
+ pending: Vec::new(),
+ prefix: Vec::new(),
+ body: Vec::new(),
+ };
+ for unit in units(text, blocks(text)) {
+ match unit {
+ Unit::Heading { level, range } => p.heading(level, range),
+ Unit::Text(range) => p.text_unit(range),
+ Unit::Table {
+ caption,
+ head,
+ rows,
+ } => p.table_unit(caption, head, rows),
+ }
+ }
+ // 文档以标题收尾:标题自己成一块,总好过丢掉
+ if !p.pending.is_empty() {
+ p.prefix = p.prefix_now();
+ p.body = p.pending.drain(..).map(Piece::Text).collect();
+ }
+ p.flush();
+ p.out
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn table(n_rows: usize) -> String {
+ let mut s = String::from("The results of the voting were as follows:\n\n| Nominee | For | Against |\n| --- | --- | --- |\n");
+ for i in 0..n_rows {
+ s.push_str(&format!("| Director {i} | {} | {} |\n", 1000 + i, 50 + i));
+ }
+ s
+ }
+
+ /// 财报里那 13 块的形状:表比预算大,续块必须还看得见表头和说明句。
+ #[test]
+ fn a_wide_table_is_split_by_rows_and_every_piece_carries_its_header_and_caption() {
+ let text = format!("# Item 5.07\n\n{}", table(40));
+ let pieces = chunk_with_budget(&text, 120);
+ assert!(
+ pieces.len() > 2,
+ "40 行在 120 token 里必然切成好几块: {}",
+ pieces.len()
+ );
+ for (i, p) in pieces.iter().enumerate() {
+ assert!(
+ p.text.contains("| Nominee | For | Against |"),
+ "第 {i} 块没有表头:\n{}",
+ p.text
+ );
+ assert!(
+ p.text.contains("| --- | --- | --- |"),
+ "第 {i} 块没有分隔行"
+ );
+ assert!(
+ p.text
+ .contains("The results of the voting were as follows:"),
+ "第 {i} 块没有说明句"
+ );
+ assert!(
+ p.text.starts_with("# Item 5.07"),
+ "第 {i} 块没有章节面包屑:\n{}",
+ p.text
+ );
+ assert_eq!(p.heading.as_deref(), Some("Item 5.07"));
+ }
+ // 每一行只出现一次,一行不拆
+ let all: String = pieces
+ .iter()
+ .map(|p| p.text.as_str())
+ .collect::<Vec<_>>()
+ .join("\n");
+ for i in 0..40 {
+ assert_eq!(
+ all.matches(&format!("| Director {i} |")).count(),
+ 1,
+ "Director {i}"
+ );
+ }
+ }
+
+ #[test]
+ fn a_table_that_fits_stays_whole_with_its_caption() {
+ let text = format!("Intro paragraph.\n\n{}\nClosing.", table(3));
+ let pieces = chunk_with_budget(&text, 1000);
+ assert_eq!(pieces.len(), 1);
+ let t = &pieces[0].text;
+ assert!(t.contains("The results of the voting were as follows:\n\n| Nominee"));
+ assert!(t.ends_with("Closing."));
+ }
+
+ /// 说明句已经在上一块的尾巴上、表放不进去了:说明句得跟着表走,不能留在上一块。
+ #[test]
+ fn a_caption_never_parts_from_its_table() {
+ let filler = "Words about nothing in particular. ".repeat(12);
+ let text = format!("{filler}\n\n{}", table(4));
+ let pieces = chunk_with_budget(&text, 140);
+ assert!(pieces.len() >= 2);
+ let with_table = pieces
+ .iter()
+ .find(|p| p.text.contains("| Nominee |"))
+ .expect("有表的那块");
+ assert!(with_table
+ .text
+ .contains("The results of the voting were as follows:"));
+ let before: Vec<&ChunkPiece> = pieces
+ .iter()
+ .filter(|p| !p.text.contains("| Nominee |"))
+ .collect();
+ for p in before {
+ assert!(
+ !p.text.contains("were as follows"),
+ "说明句留在了没有表的块里:\n{}",
+ p.text
+ );
+ }
+ }
+
+ #[test]
+ fn a_chunk_opens_with_the_headings_it_lives_under() {
+ let text = "# Report\n\n## Part A\n\nFirst.\n\n## Part B\n\nSecond.\n\n### B.1\n\nThird.";
+ let pieces = chunk_with_budget(text, 1000);
+ assert_eq!(pieces.len(), 1);
+ assert!(pieces[0]
+ .text
+ .starts_with("# Report\n\n## Part A\n\nFirst."));
+ assert_eq!(pieces[0].heading.as_deref(), Some("Report › Part B › B.1"));
+
+ let pieces = chunk_with_budget(text, 14);
+ let third = pieces.iter().find(|p| p.text.contains("Third.")).unwrap();
+ assert!(
+ third.text.starts_with("# Report\n\n## Part B\n\n### B.1"),
+ "{}",
+ third.text
+ );
+ assert_eq!(third.heading.as_deref(), Some("Report › Part B › B.1"));
+ assert_eq!(
+ &text[third.char_start as usize..third.char_end as usize],
+ "### B.1\n\nThird."
+ );
+ }
+
+ #[test]
+ fn a_long_paragraph_still_splits_by_sentence_and_offsets_are_verbatim() {
+ let text = "One sentence here. Another sentence follows it. ".repeat(30);
+ let pieces = chunk_with_budget(&text, 40);
+ assert!(pieces.len() > 1);
+ for p in &pieces {
+ assert_eq!(&text[p.char_start as usize..p.char_end as usize], p.text);
+ assert!(tokens(&p.text) <= 40);
+ }
+ }
+
+ #[test]
+ fn chunks_are_numbered_from_zero_and_never_empty() {
+ let text = "# T\n\nA.\n\n* * *\n\nB.\n\n| x | y |\n| --- | --- |\n| 1 | 2 |";
+ let pieces = chunk_text(text);
+ assert_eq!(pieces.len(), 1);
+ for (i, p) in pieces.iter().enumerate() {
+ assert_eq!(p.seq as usize, i);
+ assert!(!p.text.trim().is_empty());
+ }
+ assert!(!pieces[0].text.contains("* * *"), "水平线不进块");
+ }
+
+ /// 新闻稿末尾的页码「4」:不单独成块,并进上一块,哪怕上一块已经满了。
+ #[test]
+ fn a_page_number_never_becomes_a_chunk_of_its_own() {
+ let body = "A sentence that fills the budget nicely. ".repeat(6);
+ let text = format!(
+ "{body}
+
+4"
+ );
+ let pieces = chunk_with_budget(&text, tokens(body.trim()));
+ assert_eq!(
+ pieces.len(),
+ 1,
+ "{:?}",
+ pieces.iter().map(|p| &p.text).collect::<Vec<_>>()
+ );
+ assert!(pieces[0].text.ends_with(
+ "
+
+4"
+ ));
+ }
+
+ /// 第五项议案的形状:说明句 · 分页符 · 表。分页符不隔开说明句和它的表。
+ #[test]
+ fn a_page_break_between_a_caption_and_its_table_does_not_part_them() {
+ let filler = "Words about nothing in particular. ".repeat(10);
+ let text = format!(
+ "{filler}\n\n5. Stockholders did not approve the proposal. The results of the voting were as follows:\n\n* * *\n\n| Number of shares For | 144 |\n| --- | --- |\n| Number of shares Against | 16 |\n"
+ );
+ let pieces = chunk_with_budget(&text, 110);
+ let table = pieces
+ .iter()
+ .find(|p| p.text.contains("| Number of shares For |"))
+ .expect("有表的那块");
+ assert!(
+ table
+ .text
+ .contains("The results of the voting were as follows:"),
+ "{}",
+ table.text
+ );
+ for p in pieces
+ .iter()
+ .filter(|p| !p.text.contains("| Number of shares For |"))
+ {
+ assert!(
+ !p.text.contains("were as follows"),
+ "说明句留在了没有表的块里:\n{}",
+ p.text
+ );
+ }
+ }
+
+ /// 十位董事的形状:一句说明、一串表。每张表不论落在哪一块都看得见那句说明;
+ /// 同一块里的几张表,说明只念一遍。
+ #[test]
+ fn one_caption_introduces_the_whole_run_of_tables_after_it() {
+ let mut text = String::from("Stockholders elected each of the director nominees. The results of the voting were as follows:\n\n");
+ for (i, name) in [
+ "Tench Coxe",
+ "John Dabiri",
+ "Jen-Hsun Huang",
+ "Dawn Hudson",
+ "Harvey Jones",
+ ]
+ .iter()
.enumerate()
- .map(|(i, (offset, chunk))| ChunkPiece {
- seq: i as i32,
- text: chunk.to_string(),
- char_start: offset as i32,
- char_end: (offset + chunk.len()) as i32,
- })
- .collect()
+ {
+ text.push_str(&format!(
+ "| {name} | |\n| --- | --- |\n| Number of shares For | {} |\n| Number of shares Against | {} |\n\n",
+ 1000 + i,
+ 50 + i
+ ));
+ }
+ text.push_str(
+ "Another matter entirely.\n\n| Unrelated | 1 |\n| --- | --- |\n| row | 2 |\n",
+ );
+ let pieces = chunk_with_budget(&text, 90);
+ assert!(pieces.len() >= 3, "{}", pieces.len());
+ let huang = pieces
+ .iter()
+ .find(|p| p.text.contains("Jen-Hsun Huang"))
+ .expect("黄仁勋那块");
+ assert!(
+ huang.text.contains("director nominees"),
+ "续块丢了说明句:\n{}",
+ huang.text
+ );
+ for p in &pieces {
+ assert!(
+ p.text.matches("were as follows").count() <= 1,
+ "同一块里说明念了不止一遍:\n{}",
+ p.text
+ );
+ }
+ // 一串到别的东西出现为止:后面那张无关的表不带这句说明
+ let unrelated = pieces
+ .iter()
+ .find(|p| p.text.contains("| Unrelated |"))
+ .unwrap();
+ let before = &unrelated.text[..unrelated.text.find("| Unrelated |").unwrap()];
+ assert!(!before.ends_with("follows:\n\n"), "{}", unrelated.text);
+ assert!(unrelated.text.contains("Another matter entirely."));
+ }
+
+ #[test]
+ fn a_rule_is_not_a_boundary() {
+ let text = "Alpha.\n\n* * *\n\nBeta.";
+ assert_eq!(chunk_with_budget(text, 1000).len(), 1);
+ }
}
diff --git a/crates/utopia-ingest/src/lib.rs b/crates/utopia-ingest/src/lib.rs
index 58e8f68a..c05b9715 100644
--- a/crates/utopia-ingest/src/lib.rs
+++ b/crates/utopia-ingest/src/lib.rs
@@ -105,12 +105,13 @@ mod html_tests {
}
}
+mod blocks;
mod chunker;
pub mod html;
pub mod ontology_rdf;
mod parsers;
-pub use chunker::{chunk_text, ChunkPiece};
+pub use chunker::{chunk_text, chunk_with_budget, ChunkPiece, BUDGET_TOKENS};
/// Decode fetched text with the same encoding detection as file ingestion.
pub use parsers::plain_text as decode_text;
diff --git a/crates/utopia-server/src/pipeline.rs b/crates/utopia-server/src/pipeline.rs
index 56ce766b..590a0f30 100644
--- a/crates/utopia-server/src/pipeline.rs
+++ b/crates/utopia-server/src/pipeline.rs
@@ -65,7 +65,7 @@ async fn run(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {
let text_len = parsed.text.chars().count() as i32;
// 2. 分块 + 入库
- let pieces = utopia_ingest::chunk_text(&parsed.text);
+ let pieces = utopia_ingest::chunk_with_budget(&parsed.text, state.chunk_tokens);
let chunk_pairs =
utopia_store::documents::replace_chunks(&state.pool, doc.kb_id, document_id, &pieces)
.await?;
diff --git a/crates/utopia-server/src/state.rs b/crates/utopia-server/src/state.rs
index f74f0041..24c37c9e 100644
--- a/crates/utopia-server/src/state.rs
+++ b/crates/utopia-server/src/state.rs
@@ -29,6 +29,8 @@ pub struct AppState {
pub open_registration: bool,
/// 强制 Secure cookie(配置项);未强制时按请求的 X-Forwarded-Proto 逐次判定
pub cookie_secure: bool,
+ /// 分块预算,见 `AppConfig::chunk_tokens`
+ pub chunk_tokens: usize,
/// worker 并发数:调度循环每轮热读——系统设置改动即时生效
pub worker_concurrency: Arc<std::sync::atomic::AtomicUsize>,
/// 按模型的并发闸门:后台任务调 LLM 前取许可。限额存库,改完即时生效
@@ -58,6 +60,7 @@ impl AppState {
blob,
open_registration: cfg.open_registration,
cookie_secure: cfg.cookie_secure,
+ chunk_tokens: cfg.chunk_tokens,
worker_concurrency: Arc::new(std::sync::atomic::AtomicUsize::new(32)),
model_gates: Arc::new(crate::llm_util::ModelGates::default()),
events,
diff --git a/crates/utopia-store/src/documents.rs b/crates/utopia-store/src/documents.rs
index d48597be..652e9214 100644
--- a/crates/utopia-store/src/documents.rs
+++ b/crates/utopia-store/src/documents.rs
@@ -1024,7 +1024,8 @@ pub async fn replace_chunks(
let claimed = claim_pool.get_mut(&piece.text).and_then(|ids| ids.pop());
if let Some(id) = claimed {
sqlx::query(
- "UPDATE chunks SET seq = $2, char_start = $3, char_end = $4, doc_version = $5
+ "UPDATE chunks SET seq = $2, char_start = $3, char_end = $4, doc_version = $5,
+ heading = $6
WHERE id = $1",
)
.bind(id)
@@ -1032,6 +1033,7 @@ pub async fn replace_chunks(
.bind(piece.char_start)
.bind(piece.char_end)
.bind(version)
+ .bind(&piece.heading)
.execute(&mut *tx)
.await?;
adopted.push(id);
@@ -1062,8 +1064,8 @@ pub async fn replace_chunks(
for (id, piece) in to_insert {
sqlx::query(
"INSERT INTO chunks
- (id, kb_id, document_id, seq, text, char_start, char_end, doc_version)
- VALUES ($1, $2, $3, $4, $5, $6, $7, $8)",
+ (id, kb_id, document_id, seq, text, char_start, char_end, doc_version, heading)
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)",
)
.bind(id)
.bind(kb_id)
@@ -1073,6 +1075,7 @@ pub async fn replace_chunks(
.bind(piece.char_start)
.bind(piece.char_end)
.bind(version)
+ .bind(&piece.heading)
.execute(&mut *tx)
.await?;
}
diff --git a/crates/utopia-store/tests/a_search_reads_the_base_as_it_was.rs b/crates/utopia-store/tests/a_search_reads_the_base_as_it_was.rs
index 8dfc5679..0fcf450f 100644
--- a/crates/utopia-store/tests/a_search_reads_the_base_as_it_was.rs
+++ b/crates/utopia-store/tests/a_search_reads_the_base_as_it_was.rs
@@ -106,6 +106,7 @@ async fn seed(pool: &PgPool) -> anyhow::Result<Fixture> {
text: "The 2026 handbook says the office moved to Shenzhen.".into(),
char_start: 0,
char_end: 51,
+ heading: None,
}];
utopia_store::documents::replace_chunks(pool, kb, doc, &pieces).await?;
let new_chunk: (Uuid,) = sqlx::query_as(
diff --git a/docs/decisions/0039-a-chunk-is-what-extraction-sees.md b/docs/decisions/0039-a-chunk-is-what-extraction-sees.md
new file mode 100644
index 00000000..16fb6973
--- /dev/null
+++ b/docs/decisions/0039-a-chunk-is-what-extraction-sees.md
@@ -0,0 +1,240 @@
+# 0039 · A chunk is what extraction sees
+
+- **Status**: Implemented, cut 1 · the parser's Markdown is read into top-level blocks
+ (`blocks.rs`), the packer works over blocks with a token budget (`chunker.rs`): a table
+ travels with its caption and header, a table too wide for one chunk is split by rows and every
+ continuation repeats caption and header, headings become a breadcrumb prefix and the
+ `chunks.heading` column, a table split by a page break is joined back · not in this cut:
+ an external parser behind the block model (cut 2, Docling for PDF layout and scans), evidence
+ that points at a table cell or an image region, a budget measured for its own sake
+- **Written**: 2026-09-13 (conventions in the [README](README.md))
+- **Related**: [0006](0006-ontology-scale-and-the-prompt.md) sets the other half of the prompt
+ budget; [0033](0033-rss-source-summaries-are-source-scoped.md) is the recall bench's home;
+ the recall bench (`scripts/bench/recall.mjs`) is the instrument this record was measured with
+
+> Four SEC filings, a 52-item truth table read off the originals by a person. Two rounds on
+> `dev` on 2026-09-13 scored 43 and 45. Relation edges: 12/12 both times. Every miss was a
+> literal — a price, a date, a title, a vote count — and the misses barely overlapped between
+> rounds. The extractor was not failing to read the sentences; it was being handed tables it
+> could not read.
+
+## The problem
+
+The chunker was twenty-nine lines: a character budget of 1200, an overlap of 150, boundaries
+chosen by a generic text splitter that knows about paragraphs and sentences and nothing else.
+Three things followed, all visible in the chunks stored for the bench corpus:
+
+**A table's continuation lost its header.** Of the earnings release's 35 chunks, 13 began in the
+middle of a table. The chunk that holds the non-GAAP EPS opens with `| | Non-GAAP* | | $ | 2.22 |
+| | $ | 1.87 | …` — five dollar figures, no column names. Which quarter is $2.22? The model
+guesses. Sometimes it guesses right, which is why that item was a miss in one round and a hit
+in the next, and why the score moved by two between rounds with the same code.
+
+**A table lost its caption.** "The results of the voting were as follows:" sat at the end of one
+chunk and the votes it introduced started the next. The sentence that says what the numbers are
+was not in the chunk with the numbers.
+
+**A page break split a table, and the splitter took the break as a boundary.** The 8-K's HTML has
+an `<hr>` where the paper page turned, mid-table for Stephen C. Neal's votes. The converter
+renders it as a thematic break, the splitter prefers thematic breaks as split points, and the
+second half — abstentions and broker non-votes — became a table with no name on it.
+
+Underneath all three: the budget said 1200 characters and the comment beside it said "about
+1000+ tokens". That is true of Chinese. English is about 300 tokens in 1200 characters, so a
+wide table never fit, and the same document got a quarter of the context in one language that
+it got in the other.
+
+## Dead ends
+
+**A smarter splitter alone.** `text-splitter` has a `MarkdownSplitter` that ranks headings above
+paragraphs above sentences. It was the obvious first move and it does not do the thing that
+matters: it ranks a table row at the same level as a table, so a table that exceeds the budget
+is split between rows exactly as before, and nothing repeats the header. It also keeps thematic
+breaks as a preferred boundary, which is the Neal failure. What it does give — heading
+attachment, token sizing — this cut takes, but as a fallback for one oversized paragraph, not
+as the chunker.
+
+**Overlap as context.** The old 150-character overlap was meant to carry context across a
+boundary. For prose it carries the tail of a sentence; for a table it carries the tail of the
+previous row, which is the wrong information. Context is a header and a caption, and those have
+to be chosen, not copied from wherever the previous chunk happened to end.
+
+**An external document parser now.** Docling and Unstructured parse a document into structured
+elements and their chunkers repeat table headers on continuation. For HTML, Markdown, DOCX and
+spreadsheets our parser already produces the structure; adding a Python service to a Rust
+pipeline for a rule we can write in fifty lines buys nothing here. Where those parsers earn
+their cost is PDF layout and scanned pages, and that is cut 2 — behind the same block model,
+so the chunker does not change when the parser does.
+
+**Chunking the parser's output as blocks, without the join.** Reading the Markdown into blocks
+fixes the header and caption. It does not fix Neal: the parser had already produced two tables,
+the second with a separator row under a row of data, and a block reader faithfully reports two
+tables. The join is a separate rule and it is deliberately narrow (below).
+
+## Decisions
+
+### 1. A chunk is what extraction sees, so its boundary is part of the extraction contract
+
+The model is handed one chunk and nothing else; what the boundary cut away cannot be recovered
+by the prompt. So the chunker's rules are written from the extractor's side: never split a table
+row; never separate a table's header from its body; never separate a caption from its table;
+never separate a heading from the first block under it. These are not typesetting preferences.
+Each one, when broken, produced a specific miss in the bench.
+
+### 2. The parser's text is read into blocks, and the packer works over blocks
+
+`blocks.rs` reads the parser's Markdown with `pulldown-cmark` into a sequence of top-level
+blocks — heading, paragraph, table (header lines and body rows as separate ranges), rule, other
+— each carrying its byte range in the source. `chunker.rs` packs blocks. The split into two
+layers is what makes cut 2 cheap: an external parser produces blocks and the packer does not
+know or care.
+
+Chunk text is assembled from verbatim slices of the source. Nothing is rewritten, so a quote the
+model gives back is still found in the chunk by `span_in_quote`.
+
+### 3. A table travels with its caption and its header, and a continuation repeats both
+
+A short paragraph immediately before a table — at most 200 bytes, or ending in a colon — is the
+table's caption and moves with it. A table that fits goes in whole. One that does not is split
+between rows, and every piece is caption, header lines, then as many rows as fit. A row is never
+split; a piece that is caption plus header plus one row and still over budget is emitted anyway,
+because there is no better cut. This is the change that addresses the 13 headerless chunks
+directly. The rest of this record makes it possible; this is the point.
+
+> **Revised 2026-09-13, after the funded 300-token round.** Two gaps in "a caption moves with its
+> table", both visible in one filing. *A caption introduces the run of tables after it, not the
+> first one.* "Stockholders approved the election of each of our ten director nominees. The
+> results of the voting were as follows:" is followed by ten tables; the caption travelled with
+> Tench Coxe's, and Jen-Hsun Huang's table sat in another chunk with four correct vote counts and
+> nothing saying what the vote was for — the "elected director" edge went. A caption now attaches
+> to every table that follows it until something other than a table or a rule appears, and within
+> one chunk it is rendered once. *A rule between a caption and its table does not part them.*
+> Proposal 5's "results were as follows:" and its table are a page apart in the HTML; the lookahead
+> saw the rule and not the table, and the caption stayed in the previous chunk. Rules are skipped
+> when looking for a caption's tables. Both are rules about document structure, not about SEC
+> wording.
+
+### 4. The budget is tokens, not characters — and it is 300, because 1000 was measured and lost
+
+> **Revised 2026-09-13, the same day, by the first bench round.** The paragraph below chose
+> 1000 to honour the old comment. One round at 1000 scored 36/52 against 43 and 45 before, and
+> the acquisition 8-K showed why: nine chunks became two, and from 4,700 characters the model
+> wrote seven facts — the address, the phone number, "published in the SEC" — and not one about
+> the acquisition. The extractor's output per call does not grow with its input; handed more, it
+> picks the easiest few. The default is now 300, which is what the old character budget came to
+> in English and the condition the 43/45 rounds were measured under. Chinese gets less context
+> than it did; that is unmeasured and open. The budget is a knob (`UTOPIA_CHUNK_TOKENS`) so the
+> next measurement does not need a rebuild.
+
+Counted with `tiktoken` cl100k, which is embedded in the crate and needs no network. It is not
+the extraction model's tokenizer — that is DeepSeek's — but it is a BPE with a similar ratio,
+and the budget is a ceiling, not an accounting. 1000 is the number the old comment said the
+character budget was meant to be. Chinese keeps roughly the context it had; English gets three
+times as much, which is what lets a wide table fit at all.
+
+Budget size is a knob with its own trade-off — fewer, larger calls against more facts per reply
+— and it was not measured on its own in this cut. The bench comparison below therefore compares
+structure and budget together. Measuring the budget alone is open.
+
+### 5. Headings are a breadcrumb, written into the chunk and into `chunks.heading`
+
+The heading path a chunk sits under is prepended to its text, as the heading lines themselves,
+and stored as plain text in `chunks.heading`, a column that had existed since the ingest
+migration and was never written. The prefix is skipped when the chunk's body already begins with
+that heading. A heading never ends a chunk on its own; it attaches to what follows.
+
+### 6. A thematic break is not a boundary, and a table split by one is joined back
+
+The old splitter preferred to break at a horizontal rule. In this corpus a horizontal rule is a
+page break, and a page break is where a table is most likely to be cut in half. Rules are dropped
+from the block sequence and do not influence packing.
+
+Separately, in `blocks.rs`: the sequence *table, rule, table* where the two tables have the same
+column count and the second's "header" row begins with the same two words as one of the first's
+body rows — `Number of shares Abstaining` after `Number of shares For` — is one table. The
+second's header row becomes a body row of the first, and its separator line is dropped. The rule
+is narrow on purpose. A wider one, "same width and the header holds numbers", would join two
+financial tables whose headers are years, because `2025 | 2024` are numbers too. A test holds
+both shapes.
+
+### 7. No overlap
+
+Blocks are semantic units. Context is the breadcrumb, the caption and the header, chosen for
+what they are, not the tail of the previous chunk copied for where it happened to end. Overlap
+also doubled the facts the extractor was asked to produce on the overlapped span; the ledger
+deduplicates, but the calls were paid for.
+
+### 8. `char_start` and `char_end` name the body, not the prefix
+
+The two columns point at the verbatim span of the chunk's body in the parsed text; the prefix
+(breadcrumb, repeated caption, repeated header) is copied from elsewhere and is not part of that
+span. Nothing reads these columns today except one test fixture; this is recorded so that the
+first reader knows what they get.
+
+### 9. Every document is re-chunked on its next reprocess, and nothing is done in advance
+
+`replace_chunks` claims old chunks by exact text. New text for every chunk means every chunk is
+superseded and re-inserted, and the document re-extracted. That is the existing reprocess path;
+old chunks keep their vectors and their evidence links as before. There is no migration to
+re-chunk the whole deployment: the old chunks are not wrong, they are worse, and the next
+reprocess of each document is when they improve.
+
+## What the bench found (2026-09-13)
+
+Before, two rounds on `dev` unchanged: **43/52** and **45/52**; relations 12/12 in both; the
+stable misses across both rounds were three, none of them present in `extraction_drops` or
+`ontology_misses` — never extracted, not extracted and dropped.
+
+After, with `--reprocess` so every chunk is rebuilt. At 1000 tokens: **36/52**, relations 8/12,
+the acquisition 8-K 4/11 with seven facts from two chunks — the round that overturned decision 4.
+At 300 tokens: **37/52**, but not a measurement: the extraction model's account ran out of
+balance (HTTP 402) at 01:22 while the Ohio exhibit was being extracted, and its chunks 3–7 —
+the paragraph holding all seven of its numbers — were never sent. On the three documents that
+finished before the balance ran out the round scored 29/37, against 29/37 and 33/37 for the two
+rounds before; within one standard deviation of both. The round is to be rerun once the account
+is funded, and the number that matters is the Ohio exhibit's.
+
+Rerun at 300 tokens with the account funded: **45/52**. The total equals the better baseline round,
+but the distribution moved: literals **36/39** (baseline 31 and 32), the Ohio exhibit **15/15**,
+relations **8/12** (baseline 12 and 12). Of the four relation misses, three are the Vera Rubin
+partners, and they are in the graph — the model wrote `NVIDIA partner CoreWeave` and four more
+where the truth reads the platform as the subject; both readings are defensible and this one had
+flipped by chance in the baseline rounds. The fourth, Huang's election, was structural, and it is
+the first revision to decision 3 above.
+
+Two rounds on the integration of this change with the caption-run revision and the structural
+shape checks of #637, extracting the same rebuilt chunks: **43/52**, then **46/52** once the
+shape checks read structure instead of words. The second round had relations **12/12** and the
+earnings release **19/19**. Its six misses were four Ohio literals, the acquisition's fiscal
+year end and one vote count. None of them appears in `extraction_drops`, so the model did not
+write them; the Ohio exhibit's literals are the figures that move most between rounds (15/15,
+12/15, 11/15). Every round since the change lies within one standard deviation of the baseline,
+with no headerless continuation chunk in any of them.
+
+Seen in the rebuilt chunks, independent of the score: no chunk begins inside a table without
+its header (was 13 of 35 in the earnings release); the Neal vote table is one table again with
+no rule inside it; the chunk holding the non-GAAP EPS carries a header — the parser's layout row,
+not the column names, which is the open question below.
+
+## Open questions
+
+- **The budget on its own.** 1000 tokens was chosen to honour the old comment, not measured.
+ The same bench at 300 tokens (character parity for English) and at 1000 would separate the
+ structural change from the size change. Two runs of twenty minutes each.
+- **Evidence that points at a cell or a region.** A fact's evidence is a text span. For a table
+ it should be able to name a row and a column; for a figure, a region on a page. That is the
+ precondition for extracting from images at all and it is the same ledger discipline —
+ document, evidence, fact — extended to two more kinds of evidence. Not this record.
+- **The header the parser hands over.** The chunker repeats whatever the parser marked as the
+ header. In the XBRL earnings release that is `| | NVIDIA CORPORATION | | … |` — a layout row
+ promoted because the table had no `<th>` — while the rows that actually name the columns
+ (`Three Months Ended | Six Months Ended`, then the dates) are the first body rows and stay in
+ the first piece. Seen on the re-chunked corpus: the chunk holding the non-GAAP EPS now opens
+ with a header and still does not say which quarter is which. The mechanism is right and the
+ input is wrong; choosing the header rows is `promote_first_row_headers`' job and belongs with
+ the `head_blank` rule already there. Not this cut.
+- **DOCX and PDF tables.** The DOCX parser flattens tables to paragraphs; the PDF parser has no
+ tables. Both arrive at the block reader as prose and get prose treatment. Spreadsheets and CSV
+ come out as tab- and pipe-separated lines rather than Markdown tables and are likewise read as
+ paragraphs. Emitting real Markdown tables from those four parsers is cheap for the first two
+ and is cut 2 for the last two.
diff --git a/docs/decisions/README.md b/docs/decisions/README.md
index b92bdbae..1b75ab03 100644
--- a/docs/decisions/README.md
+++ b/docs/decisions/README.md
@@ -62,6 +62,7 @@ The test for writing one: if someone (including us) looks at a piece of code in
| 0036 | [Exploration aligns a schema to the ontology](0036-exploration-aligns-a-schema-to-the-ontology.md) | Written, not implemented · exploration proposes an alignment per table (the class it is a table of, its attributes, its relations, a conversion tree per column) through `ontology_proposals`, adopted as one thing · a definition is a rule over aligned attributes, written by a person; a shared convention is one rule others read · conversions are 0032's expression tree, text parsed by `sqlparser`, never stored as SQL · `Metric` / `Dimension` retire and `concept_mappings` becomes rendered · the schema document leaves extraction · where a rule runs against the source stays open |
| 0037 | [A relation carries its own attributes](0037-a-relation-carries-its-own-attributes.md) | Written · cut 1 in progress: qualifier tables (0049), declaration, extraction, panel, export · entity-valued qualifier reserved, conflict-as-two-rows and the canvas label next |
| 0038 | [The interface has a light side](0038-the-interface-has-a-light-side.md) | Implemented (#599) · `data-theme` on `<html>`, chosen by the reader and stored in the browser, `system` resolved to one of the two before first paint · a second token block in `styles.css` and not one rule rewritten: alpha stays, the triplet swaps · the canvas reads its colours from the tokens through one reader file and rebuilds on a switch; edges are flattened over the ground on light because the WebGL shader can only brighten · a `raw-colour` guard rule over `.ts` and `.tsx` · the entity palette is the same in both themes; a paper-tuned set is open |
+| 0039 | [A chunk is what extraction sees](0039-a-chunk-is-what-extraction-sees.md) | Implemented, cut 1 · the parser's Markdown is read into blocks and packed by a token budget (1000, cl100k) · a table travels with its caption and header and a continuation repeats both · headings become a breadcrumb and fill `chunks.heading` · a page-break rule is not a boundary and a table it split is joined back · not measured on its own: the budget · cut 2: an external parser behind the block model, evidence that names a cell or a region |
## Not a decision record
diff --git a/scripts/bench/recall.mjs b/scripts/bench/recall.mjs
index 7cffe2a8..faf901ad 100644
--- a/scripts/bench/recall.mjs
+++ b/scripts/bench/recall.mjs
@@ -202,7 +202,8 @@ psql(`DELETE FROM extraction_drops WHERE kb_id='${KB}'`);
psql(`DELETE FROM ontology_misses WHERE kb_id='${KB}'`);
// 自动扩本体上一轮长出来的关系:留着就进下一轮的提示词,两轮条件不同
psql(`DELETE FROM relation_types WHERE kb_id='${KB}' AND created_at > '${packTs}'::timestamptz + interval '1 second'`);
-psql(`UPDATE knowledge_bases SET auto_extend_ontology=FALSE, auto_type_resolution=FALSE WHERE id='${KB}'`);
+// 四个会在抽取之后改图的开关一律关掉:测的是抽取本身,推理物化和治理闸门会在打分前增删事实
+psql(`UPDATE knowledge_bases SET auto_extend_ontology=FALSE, auto_type_resolution=FALSE, materialize_inferences=FALSE, governance=FALSE WHERE id='${KB}'`);
psql(`UPDATE chunks SET extracted_at=NULL WHERE document_id IN (SELECT id FROM documents WHERE kb_id='${KB}' AND filename IN (${names}))`);
console.log(`本体 ${num(`SELECT count(*) FROM relation_types WHERE kb_id='${KB}'`)} 个关系 / ${num(`SELECT count(*) FROM entity_types WHERE kb_id='${KB}'`)} 个类`);