Skip to content

World's first open-source enterprise world model.

A Word paragraph ends a Markdown block, so split tables keep their captions (#1038)

Coderfudb36f3fce135
diff --git a/crates/utopia-ingest/src/parsers.rs b/crates/utopia-ingest/src/parsers.rs
index 38c8c88d..c60e7fa9 100644
--- a/crates/utopia-ingest/src/parsers.rs
+++ b/crates/utopia-ingest/src/parsers.rs
@@ -133,10 +133,11 @@ fn pdf_with_poppler(bytes: &[u8]) -> anyhow::Result<String> {
String::from_utf8(output.stdout).context("pdftotext returned invalid UTF-8")
}
-/// docx:解压 word/document.xml。正文 w:t 取字、w:p 分段;表格(w:tbl)收成网格交给
-/// `table::render_grid`,和 HTML 表走同一套渲染:w:gridSpan 跨列,上下合并(w:vMerge)的
-/// 续格留空,格内段落的左缩进 w:ind 当内边距(小节行靠它折进标签)。套在格子里的表按格子
-/// 文字处理。标题(有大纲级别的段落,见 [`docx_heading_styles`])写成 Markdown 标题。
+/// docx:解压 word/document.xml。正文 w:t 取字;不在表格里的 w:p 是一段,段末留空行;
+/// 段内的 w:br/w:cr 仍是单换行。表格(w:tbl)收成网格交给 `table::render_grid`,和 HTML 表
+/// 走同一套渲染:w:gridSpan 跨列,上下合并(w:vMerge)的续格留空,格内段落的左缩进 w:ind
+/// 当内边距(小节行靠它折进标签)。套在格子里的表按格子文字处理。标题(有大纲级别的段落,
+/// 见 [`docx_heading_styles`])写成 Markdown 标题。
pub fn docx(bytes: &[u8]) -> anyhow::Result<String> {
let mut archive =
zip::ZipArchive::new(Cursor::new(bytes)).context("Malformed docx structure")?;
@@ -254,9 +255,12 @@ pub(crate) fn docx_xml_to_text(
let mut cell: Option<(String, usize, u32)> = None;
// 正文里的一段(不在表格里、不是文本框里套着的段)是不是标题。`paragraphs` 数套了几层
// w:p,`para_start` 是这一段在 out 里开始的位置。级别先看段落自己写的 w:outlineLvl,
- // 再看它的样式;两样都只认第一次出现的,w:pPrChange 里记的是修订之前的样式,不算
+ // 再看它的样式;两样都只认第一次出现的,w:pPrChange 里记的是修订之前的样式,不算。
+ // 另一个栈记每段开始的位置,用来在段末把空段丢掉、非空段之间只留一个空行。文本框会
+ // 在宿主段里再套 w:p,栈让内层段落也能各自收尾,而标题判定仍只看最外层。
let mut paragraphs = 0usize;
let mut para_start = 0usize;
+ let mut paragraph_starts: Vec<usize> = Vec::new();
let mut own_level: Option<Option<u8>> = None;
let mut style_level: Option<Option<u8>> = None;
let mut in_revision = false;
@@ -333,6 +337,9 @@ pub(crate) fn docx_xml_to_text(
"w:t" => in_text = true,
"w:p" => {
paragraphs += 1;
+ if cell.is_none() {
+ paragraph_starts.push(out.len());
+ }
if paragraphs == 1 && cell.is_none() {
para_start = out.len();
own_level = None;
@@ -387,11 +394,12 @@ pub(crate) fn docx_xml_to_text(
match cell.as_mut() {
Some(c) => c.0.push(' '),
None => {
+ let start = paragraph_starts.pop().unwrap_or(out.len());
// 标题写成一行 Markdown 标题:分块器靠它给每块开头补上所在的各级标题,
// 时间解释靠块里的标题行分节(0064 决定 1 按 cut 2 修订的那段)。没有
// 它,一份 Word 里各节的日期都算成第一节的
let level = own_level.unwrap_or(style_level.flatten());
- if let Some(level) = level.filter(|_| paragraphs == 1) {
+ if let Some(level) = level.filter(|_| paragraph_starts.is_empty()) {
let title = out[para_start..]
.split_whitespace()
.collect::<Vec<_>>()
@@ -403,7 +411,15 @@ pub(crate) fn docx_xml_to_text(
out.push_str(&title);
}
}
- out.push('\n');
+ // 一个 Word 段落是一个 Markdown 段。收尾时先去掉段内末尾的空白,
+ // 空段不留痕;非空段只补一个空行,w:br 产生的段内单换行不受影响。
+ let end = start + out[start..].trim_end().len();
+ if end == start {
+ out.truncate(start);
+ } else {
+ out.truncate(end);
+ out.push_str("\n\n");
+ }
}
}
paragraphs = paragraphs.saturating_sub(1);
@@ -418,7 +434,11 @@ pub(crate) fn docx_xml_to_text(
depth = depth.saturating_sub(1);
if depth == 0 {
if let Some(md) = crate::table::render_grid(&rows, false) {
- out.push('\n');
+ if !out.is_empty() {
+ let end = out.trim_end().len();
+ out.truncate(end);
+ out.push_str("\n\n");
+ }
out.push_str(&md);
out.push_str("\n\n");
}
@@ -1059,14 +1079,14 @@ mod tests {
#[test]
fn a_docx_table_is_rendered_under_its_headings() {
let text = docx_xml_to_text(DOC, &Default::default()).unwrap();
- assert!(text.starts_with("Segment results\n"), "{text}");
+ assert!(text.starts_with("Segment results\n\n"), "{text}");
assert!(
text.contains(
"| Segment | Revenue |\n| --- | --- |\n| Cloud | $1,200 |\n| Devices | $300 |"
),
"{text}"
);
- assert!(text.ends_with("After the table.\n"), "{text}");
+ assert!(text.ends_with("After the table.\n\n"), "{text}");
}
/// 套在格子里的表不单独成表:它的字算外层格子的字
diff --git a/crates/utopia-ingest/tests/docx_breaks.rs b/crates/utopia-ingest/tests/docx_breaks.rs
index 7250ba66..ba97fed6 100644
--- a/crates/utopia-ingest/tests/docx_breaks.rs
+++ b/crates/utopia-ingest/tests/docx_breaks.rs
@@ -43,19 +43,91 @@ fn explicit_cell_breaks_separate_numbers_and_words() {
}
#[test]
-fn paragraph_breaks_remain_newlines_and_formatting_runs_join() {
+fn paragraph_ends_are_blank_lines_but_explicit_breaks_are_newlines() {
for br in ["<w:br/>", "<w:cr/>", "<w:br></w:br>", "<w:cr></w:cr>"] {
let text = read(&format!(
"<w:p><w:r><w:t>Hel</w:t></w:r><w:r><w:t>lo</w:t>{br}<w:t>world</w:t></w:r></w:p>"
));
- assert_eq!(text.trim(), "Hello\nworld", "{br}");
+ assert_eq!(
+ text.trim(),
+ "Hello\nworld",
+ "a break inside one paragraph: {br}"
+ );
}
let text = read(&table(
"<w:t>Hel</w:t></w:r><w:r><w:t>lo</w:t><w:tab/><w:t>world</w:t>",
));
assert!(text.contains("| Sample | Hello world |"), "{text}");
let text = read("<w:p><w:r><w:t>first</w:t><w:tab/><w:t>line</w:t></w:r></w:p><w:p><w:r><w:t>second</w:t></w:r></w:p>");
- assert_eq!(text.trim(), "first line\nsecond");
+ assert_eq!(text.trim(), "first line\n\nsecond");
+}
+
+#[test]
+fn empty_word_paragraphs_do_not_make_empty_blocks_or_blank_runs() {
+ let text = read(concat!(
+ "<w:p><w:r><w:t>first</w:t></w:r></w:p>",
+ "<w:p/>",
+ "<w:p><w:r><w:t> </w:t></w:r></w:p>",
+ "<w:p><w:r><w:t>second</w:t></w:r></w:p>",
+ ));
+ assert_eq!(text.trim(), "first\n\nsecond");
+ assert!(!text.contains("\n\n\n"), "{text}");
+}
+
+#[test]
+fn a_long_word_list_packs_multiple_items_per_chunk() {
+ let body: String = (1..=30)
+ .map(|i| format!("<w:p><w:r><w:t>List item {i} is short.</w:t></w:r></w:p>"))
+ .collect();
+ let text = read(&body);
+ assert_eq!(text.matches("List item").count(), 30, "{text}");
+
+ let pieces = utopia_ingest::chunk_with_budget(&text, 40);
+ assert!(pieces.len() < 30, "{pieces:#?}");
+ assert!(
+ pieces
+ .iter()
+ .any(|piece| piece.text.matches("List item").count() > 1),
+ "{pieces:#?}"
+ );
+}
+
+fn long_table(rows: usize) -> String {
+ let mut table = String::from(
+ r#"<w:tbl><w:tr><w:tc><w:p><w:r><w:t>区域</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>2024 年营收</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>单位</w:t></w:r></w:p></w:tc></w:tr>"#,
+ );
+ for row in 1..=rows {
+ table.push_str(&format!(
+ "<w:tr><w:tc><w:p><w:r><w:t>区域 {row}</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>{row}00</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>万元</w:t></w:r></w:p></w:tc></w:tr>"
+ ));
+ }
+ table.push_str("</w:tbl>");
+ table
+}
+
+#[test]
+fn a_word_caption_is_repeated_for_every_piece_of_a_split_table() {
+ let analysis = "本段分析说明营业收入同比变化、客户结构和主要风险,不是表格的说明句。".repeat(4);
+ let caption = "表1 2024 年分区域营收(单位:万元)";
+ let text = read(&format!(
+ "<w:p><w:r><w:t>{analysis}</w:t></w:r></w:p><w:p><w:r><w:t>{caption}</w:t></w:r></w:p>{}",
+ long_table(40)
+ ));
+ assert!(
+ text.contains(&format!("{analysis}\n\n{caption}\n\n|")),
+ "{text}"
+ );
+
+ let pieces = utopia_ingest::chunk_text(&text);
+ let table_pieces: Vec<_> = pieces
+ .iter()
+ .filter(|piece| piece.text.contains("| --- |"))
+ .collect();
+ assert!(table_pieces.len() >= 2, "{pieces:#?}");
+ for piece in table_pieces {
+ assert!(piece.text.contains(caption), "{:#?}", piece.text);
+ assert!(!piece.text.contains(&analysis), "{:#?}", piece.text);
+ }
}
#[test]
diff --git a/crates/utopia-ingest/tests/docx_headings.rs b/crates/utopia-ingest/tests/docx_headings.rs
index 2e0fd299..e1599566 100644
--- a/crates/utopia-ingest/tests/docx_headings.rs
+++ b/crates/utopia-ingest/tests/docx_headings.rs
@@ -74,11 +74,11 @@ fn reports() -> String {
fn a_paragraph_with_an_outline_level_is_a_heading() {
let text = read(&reports(), Some(STYLES));
assert!(
- text.contains("# 第一周周报(2024-03-01)\n上周销售额增长 5%。\n"),
+ text.contains("# 第一周周报(2024-03-01)\n\n上周销售额增长 5%。\n"),
"{text}"
);
assert!(
- text.contains("# 第二周周报 (2024-03-08)\n## 明细\n上周退货 3 单。\n"),
+ text.contains("# 第二周周报 (2024-03-08)\n\n## 明细\n\n上周退货 3 单。\n"),
"{text}"
);
assert!(text.contains("# 附录\n"), "{text}");
diff --git a/crates/utopia-ingest/tests/docx_text_boxes.rs b/crates/utopia-ingest/tests/docx_text_boxes.rs
index 94ed4d55..672381c0 100644
--- a/crates/utopia-ingest/tests/docx_text_boxes.rs
+++ b/crates/utopia-ingest/tests/docx_text_boxes.rs
@@ -62,7 +62,7 @@ fn a_text_box_is_read_once() {
"{text}"
);
assert!(text.contains("Before the box."), "{text}");
- assert!(text.contains("After the box.\nNext paragraph."), "{text}");
+ assert!(text.contains("After the box.\n\nNext paragraph."), "{text}");
}
}
diff --git a/docs/design/sources.md b/docs/design/sources.md
index 274071d8..3454c199 100644
--- a/docs/design/sources.md
+++ b/docs/design/sources.md
@@ -35,7 +35,10 @@ table travels with its caption and header and a continuation repeats both; a cap
every table that follows it; a row is never split; headings become a breadcrumb prefix and
`chunks.heading`; a thematic break is not a boundary and a table it split is joined back; no
overlap; chunk text is verbatim slices so a quote is found by offset; a document is re-chunked on
-its next reprocess [0039]. A DOCX paragraph outside a table is a heading when it has an outline
+its next reprocess [0039]. A Word paragraph outside a table ends a Markdown paragraph, so the
+caption and packing rules see the same boundaries as Markdown and HTML: a blank line follows it, a
+`w:br` inside it stays a single newline, a paragraph inside a table cell stays a space, and empty
+paragraphs disappear [#1024]. A DOCX paragraph outside a table is a heading when it has an outline
level: its own `w:outlineLvl`, else its style's, else a style named `heading N`, else the style it
is based on (style ids differ by the language of Word, the names do not); so a Word file's
sections reach the chunks and the section dates of 0064.