World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
A Word paragraph ends a Markdown block, so split tables keep their captions (#1038)
Coderfudb36f3fce135
diff --git a/crates/utopia-ingest/src/parsers.rs b/crates/utopia-ingest/src/parsers.rsindex 38c8c88d..c60e7fa9 100644--- a/crates/utopia-ingest/src/parsers.rs+++ b/crates/utopia-ingest/src/parsers.rs@@ -133,10 +133,11 @@ fn pdf_with_poppler(bytes: &[u8]) -> anyhow::Result<String> {String::from_utf8(output.stdout).context("pdftotext returned invalid UTF-8")}-/// docx:解压 word/document.xml。正文 w:t 取字、w:p 分段;表格(w:tbl)收成网格交给-/// `table::render_grid`,和 HTML 表走同一套渲染:w:gridSpan 跨列,上下合并(w:vMerge)的-/// 续格留空,格内段落的左缩进 w:ind 当内边距(小节行靠它折进标签)。套在格子里的表按格子-/// 文字处理。标题(有大纲级别的段落,见 [`docx_heading_styles`])写成 Markdown 标题。+/// docx:解压 word/document.xml。正文 w:t 取字;不在表格里的 w:p 是一段,段末留空行;+/// 段内的 w:br/w:cr 仍是单换行。表格(w:tbl)收成网格交给 `table::render_grid`,和 HTML 表+/// 走同一套渲染:w:gridSpan 跨列,上下合并(w:vMerge)的续格留空,格内段落的左缩进 w:ind+/// 当内边距(小节行靠它折进标签)。套在格子里的表按格子文字处理。标题(有大纲级别的段落,+/// 见 [`docx_heading_styles`])写成 Markdown 标题。pub fn docx(bytes: &[u8]) -> anyhow::Result<String> {let mut archive =zip::ZipArchive::new(Cursor::new(bytes)).context("Malformed docx structure")?;@@ -254,9 +255,12 @@ pub(crate) fn docx_xml_to_text(let mut cell: Option<(String, usize, u32)> = None;// 正文里的一段(不在表格里、不是文本框里套着的段)是不是标题。`paragraphs` 数套了几层// w:p,`para_start` 是这一段在 out 里开始的位置。级别先看段落自己写的 w:outlineLvl,- // 再看它的样式;两样都只认第一次出现的,w:pPrChange 里记的是修订之前的样式,不算+ // 再看它的样式;两样都只认第一次出现的,w:pPrChange 里记的是修订之前的样式,不算。+ // 另一个栈记每段开始的位置,用来在段末把空段丢掉、非空段之间只留一个空行。文本框会+ // 在宿主段里再套 w:p,栈让内层段落也能各自收尾,而标题判定仍只看最外层。let mut paragraphs = 0usize;let mut para_start = 0usize;+ let mut paragraph_starts: Vec<usize> = Vec::new();let mut own_level: Option<Option<u8>> = None;let mut style_level: Option<Option<u8>> = None;let mut in_revision = false;@@ -333,6 +337,9 @@ pub(crate) fn docx_xml_to_text("w:t" => in_text = true,"w:p" => {paragraphs += 1;+ if cell.is_none() {+ paragraph_starts.push(out.len());+ }if paragraphs == 1 && cell.is_none() {para_start = out.len();own_level = None;@@ -387,11 +394,12 @@ pub(crate) fn docx_xml_to_text(match cell.as_mut() {Some(c) => c.0.push(' '),None => {+ let start = paragraph_starts.pop().unwrap_or(out.len());// 标题写成一行 Markdown 标题:分块器靠它给每块开头补上所在的各级标题,// 时间解释靠块里的标题行分节(0064 决定 1 按 cut 2 修订的那段)。没有// 它,一份 Word 里各节的日期都算成第一节的let level = own_level.unwrap_or(style_level.flatten());- if let Some(level) = level.filter(|_| paragraphs == 1) {+ if let Some(level) = level.filter(|_| paragraph_starts.is_empty()) {let title = out[para_start..].split_whitespace().collect::<Vec<_>>()@@ -403,7 +411,15 @@ pub(crate) fn docx_xml_to_text(out.push_str(&title);}}- out.push('\n');+ // 一个 Word 段落是一个 Markdown 段。收尾时先去掉段内末尾的空白,+ // 空段不留痕;非空段只补一个空行,w:br 产生的段内单换行不受影响。+ let end = start + out[start..].trim_end().len();+ if end == start {+ out.truncate(start);+ } else {+ out.truncate(end);+ out.push_str("\n\n");+ }}}paragraphs = paragraphs.saturating_sub(1);@@ -418,7 +434,11 @@ pub(crate) fn docx_xml_to_text(depth = depth.saturating_sub(1);if depth == 0 {if let Some(md) = crate::table::render_grid(&rows, false) {- out.push('\n');+ if !out.is_empty() {+ let end = out.trim_end().len();+ out.truncate(end);+ out.push_str("\n\n");+ }out.push_str(&md);out.push_str("\n\n");}@@ -1059,14 +1079,14 @@ mod tests {#[test]fn a_docx_table_is_rendered_under_its_headings() {let text = docx_xml_to_text(DOC, &Default::default()).unwrap();- assert!(text.starts_with("Segment results\n"), "{text}");+ assert!(text.starts_with("Segment results\n\n"), "{text}");assert!(text.contains("| Segment | Revenue |\n| --- | --- |\n| Cloud | $1,200 |\n| Devices | $300 |"),"{text}");- assert!(text.ends_with("After the table.\n"), "{text}");+ assert!(text.ends_with("After the table.\n\n"), "{text}");}/// 套在格子里的表不单独成表:它的字算外层格子的字diff --git a/crates/utopia-ingest/tests/docx_breaks.rs b/crates/utopia-ingest/tests/docx_breaks.rsindex 7250ba66..ba97fed6 100644--- a/crates/utopia-ingest/tests/docx_breaks.rs+++ b/crates/utopia-ingest/tests/docx_breaks.rs@@ -43,19 +43,91 @@ fn explicit_cell_breaks_separate_numbers_and_words() {}#[test]-fn paragraph_breaks_remain_newlines_and_formatting_runs_join() {+fn paragraph_ends_are_blank_lines_but_explicit_breaks_are_newlines() {for br in ["<w:br/>", "<w:cr/>", "<w:br></w:br>", "<w:cr></w:cr>"] {let text = read(&format!("<w:p><w:r><w:t>Hel</w:t></w:r><w:r><w:t>lo</w:t>{br}<w:t>world</w:t></w:r></w:p>"));- assert_eq!(text.trim(), "Hello\nworld", "{br}");+ assert_eq!(+ text.trim(),+ "Hello\nworld",+ "a break inside one paragraph: {br}"+ );}let text = read(&table("<w:t>Hel</w:t></w:r><w:r><w:t>lo</w:t><w:tab/><w:t>world</w:t>",));assert!(text.contains("| Sample | Hello world |"), "{text}");let text = read("<w:p><w:r><w:t>first</w:t><w:tab/><w:t>line</w:t></w:r></w:p><w:p><w:r><w:t>second</w:t></w:r></w:p>");- assert_eq!(text.trim(), "first line\nsecond");+ assert_eq!(text.trim(), "first line\n\nsecond");+}++#[test]+fn empty_word_paragraphs_do_not_make_empty_blocks_or_blank_runs() {+ let text = read(concat!(+ "<w:p><w:r><w:t>first</w:t></w:r></w:p>",+ "<w:p/>",+ "<w:p><w:r><w:t> </w:t></w:r></w:p>",+ "<w:p><w:r><w:t>second</w:t></w:r></w:p>",+ ));+ assert_eq!(text.trim(), "first\n\nsecond");+ assert!(!text.contains("\n\n\n"), "{text}");+}++#[test]+fn a_long_word_list_packs_multiple_items_per_chunk() {+ let body: String = (1..=30)+ .map(|i| format!("<w:p><w:r><w:t>List item {i} is short.</w:t></w:r></w:p>"))+ .collect();+ let text = read(&body);+ assert_eq!(text.matches("List item").count(), 30, "{text}");++ let pieces = utopia_ingest::chunk_with_budget(&text, 40);+ assert!(pieces.len() < 30, "{pieces:#?}");+ assert!(+ pieces+ .iter()+ .any(|piece| piece.text.matches("List item").count() > 1),+ "{pieces:#?}"+ );+}++fn long_table(rows: usize) -> String {+ let mut table = String::from(+ r#"<w:tbl><w:tr><w:tc><w:p><w:r><w:t>区域</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>2024 年营收</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>单位</w:t></w:r></w:p></w:tc></w:tr>"#,+ );+ for row in 1..=rows {+ table.push_str(&format!(+ "<w:tr><w:tc><w:p><w:r><w:t>区域 {row}</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>{row}00</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>万元</w:t></w:r></w:p></w:tc></w:tr>"+ ));+ }+ table.push_str("</w:tbl>");+ table+}++#[test]+fn a_word_caption_is_repeated_for_every_piece_of_a_split_table() {+ let analysis = "本段分析说明营业收入同比变化、客户结构和主要风险,不是表格的说明句。".repeat(4);+ let caption = "表1 2024 年分区域营收(单位:万元)";+ let text = read(&format!(+ "<w:p><w:r><w:t>{analysis}</w:t></w:r></w:p><w:p><w:r><w:t>{caption}</w:t></w:r></w:p>{}",+ long_table(40)+ ));+ assert!(+ text.contains(&format!("{analysis}\n\n{caption}\n\n|")),+ "{text}"+ );++ let pieces = utopia_ingest::chunk_text(&text);+ let table_pieces: Vec<_> = pieces+ .iter()+ .filter(|piece| piece.text.contains("| --- |"))+ .collect();+ assert!(table_pieces.len() >= 2, "{pieces:#?}");+ for piece in table_pieces {+ assert!(piece.text.contains(caption), "{:#?}", piece.text);+ assert!(!piece.text.contains(&analysis), "{:#?}", piece.text);+ }}#[test]diff --git a/crates/utopia-ingest/tests/docx_headings.rs b/crates/utopia-ingest/tests/docx_headings.rsindex 2e0fd299..e1599566 100644--- a/crates/utopia-ingest/tests/docx_headings.rs+++ b/crates/utopia-ingest/tests/docx_headings.rs@@ -74,11 +74,11 @@ fn reports() -> String {fn a_paragraph_with_an_outline_level_is_a_heading() {let text = read(&reports(), Some(STYLES));assert!(- text.contains("# 第一周周报(2024-03-01)\n上周销售额增长 5%。\n"),+ text.contains("# 第一周周报(2024-03-01)\n\n上周销售额增长 5%。\n"),"{text}");assert!(- text.contains("# 第二周周报 (2024-03-08)\n## 明细\n上周退货 3 单。\n"),+ text.contains("# 第二周周报 (2024-03-08)\n\n## 明细\n\n上周退货 3 单。\n"),"{text}");assert!(text.contains("# 附录\n"), "{text}");diff --git a/crates/utopia-ingest/tests/docx_text_boxes.rs b/crates/utopia-ingest/tests/docx_text_boxes.rsindex 94ed4d55..672381c0 100644--- a/crates/utopia-ingest/tests/docx_text_boxes.rs+++ b/crates/utopia-ingest/tests/docx_text_boxes.rs@@ -62,7 +62,7 @@ fn a_text_box_is_read_once() {"{text}");assert!(text.contains("Before the box."), "{text}");- assert!(text.contains("After the box.\nNext paragraph."), "{text}");+ assert!(text.contains("After the box.\n\nNext paragraph."), "{text}");}}diff --git a/docs/design/sources.md b/docs/design/sources.mdindex 274071d8..3454c199 100644--- a/docs/design/sources.md+++ b/docs/design/sources.md@@ -35,7 +35,10 @@ table travels with its caption and header and a continuation repeats both; a capevery table that follows it; a row is never split; headings become a breadcrumb prefix and`chunks.heading`; a thematic break is not a boundary and a table it split is joined back; nooverlap; chunk text is verbatim slices so a quote is found by offset; a document is re-chunked on-its next reprocess [0039]. A DOCX paragraph outside a table is a heading when it has an outline+its next reprocess [0039]. A Word paragraph outside a table ends a Markdown paragraph, so the+caption and packing rules see the same boundaries as Markdown and HTML: a blank line follows it, a+`w:br` inside it stays a single newline, a paragraph inside a table cell stays a space, and empty+paragraphs disappear [#1024]. A DOCX paragraph outside a table is a heading when it has an outlinelevel: its own `w:outlineLvl`, else its style's, else a style named `heading N`, else the style itis based on (style ids differ by the language of Word, the names do not); so a Word file'ssections reach the chunks and the section dates of 0064.