Skip to content

World's first open-source enterprise world model.

A Word document's headings are read as headings, so its sections reach the chunks and each section is dated by its own date (#1018)

Maya Kid9792b6ed24fd
diff --git a/crates/utopia-ingest/src/parsers.rs b/crates/utopia-ingest/src/parsers.rs
index 7e537a9f..7cb97809 100644
--- a/crates/utopia-ingest/src/parsers.rs
+++ b/crates/utopia-ingest/src/parsers.rs
@@ -136,15 +136,114 @@ fn pdf_with_poppler(bytes: &[u8]) -> anyhow::Result<String> {
/// docx:解压 word/document.xml。正文 w:t 取字、w:p 分段;表格(w:tbl)收成网格交给
/// `table::render_grid`,和 HTML 表走同一套渲染:w:gridSpan 跨列,上下合并(w:vMerge)的
/// 续格留空,格内段落的左缩进 w:ind 当内边距(小节行靠它折进标签)。套在格子里的表按格子
-/// 文字处理。
+/// 文字处理。标题(有大纲级别的段落,见 [`docx_heading_styles`])写成 Markdown 标题。
pub fn docx(bytes: &[u8]) -> anyhow::Result<String> {
- let xml = read_zip_entry(bytes, "word/document.xml").context("Malformed docx structure")?;
- docx_xml_to_text(&xml)
+ let mut archive =
+ zip::ZipArchive::new(Cursor::new(bytes)).context("Malformed docx structure")?;
+ let xml =
+ read_zip_entry(&mut archive, "word/document.xml").context("Malformed docx structure")?;
+ // 样式表缺了或读不了,正文照读,只是认不出靠样式定的标题
+ let headings = read_zip_entry(&mut archive, "word/styles.xml")
+ .ok()
+ .and_then(|styles| docx_heading_styles(&styles).ok())
+ .unwrap_or_default();
+ docx_xml_to_text(&xml, &headings)
+}
+
+/// 哪些段落样式是标题、第几级(样式 id → 1–9)。
+///
+/// Word 的标题就是有大纲级别的段落:导航窗格和目录都按它列,样式定义里写 `w:outlineLvl`
+/// (从 0 数,9 是正文)。样式 id 靠不住——英文 Word 写 `Heading1`,中文 Word 写 `1`、`2`,
+/// 正文样式是 `a`——内置样式的名字倒是一律写英文的 `heading 1`。所以按这个次序认:样式
+/// 自己的 `w:outlineLvl`,其次名字是 `heading N`,再次顺着 `w:basedOn` 往上找(自定义的
+/// 标题样式多半基于内置标题)。`Title`、`toc 1` 没有大纲级别,不算标题
+pub(crate) fn docx_heading_styles(
+ xml: &str,
+) -> anyhow::Result<std::collections::HashMap<String, u8>> {
+ #[derive(Default)]
+ struct Style {
+ name: Option<String>,
+ based_on: Option<String>,
+ outline: Option<u8>,
+ }
+ let attr = |e: &quick_xml::events::BytesStart<'_>, name: &str| -> Option<String> {
+ e.attributes()
+ .flatten()
+ .find(|a| a.key.as_ref() == name)
+ .map(|a| a.value.to_string())
+ };
+ let mut reader = Reader::from_str(xml);
+ let mut styles: std::collections::HashMap<String, Style> = std::collections::HashMap::new();
+ let mut current: Option<(String, Style)> = None;
+ loop {
+ match reader.read_event() {
+ Ok(Event::Start(e)) if e.name().as_ref() == "w:style" => {
+ current = (attr(&e, "w:type").as_deref() == Some("paragraph"))
+ .then(|| attr(&e, "w:styleId"))
+ .flatten()
+ .map(|id| (id, Style::default()));
+ }
+ Ok(Event::End(e)) if e.name().as_ref() == "w:style" => {
+ if let Some((id, style)) = current.take() {
+ styles.insert(id, style);
+ }
+ }
+ Ok(Event::Start(e) | Event::Empty(e)) => {
+ if let Some((_, style)) = current.as_mut() {
+ match e.name().as_ref() {
+ "w:name" => style.name = attr(&e, "w:val"),
+ "w:basedOn" => style.based_on = attr(&e, "w:val"),
+ "w:outlineLvl" => {
+ style.outline = attr(&e, "w:val").and_then(|v| v.parse().ok());
+ }
+ _ => {}
+ }
+ }
+ }
+ Ok(Event::Eof) => break,
+ Err(e) => anyhow::bail!("XML parse error: {e}"),
+ _ => {}
+ }
+ }
+ let level_of = |id: &str| -> Option<u8> {
+ let mut id = id;
+ // basedOn 最多往上走十步:Word 自己不会链这么长,写坏了成环的也就此打住
+ for _ in 0..10 {
+ let style = styles.get(id)?;
+ if let Some(outline) = style.outline {
+ return outline_level(outline);
+ }
+ if let Some(n) = style.name.as_deref().and_then(heading_number) {
+ return Some(n);
+ }
+ id = style.based_on.as_deref()?;
+ }
+ None
+ };
+ Ok(styles
+ .keys()
+ .filter_map(|id| Some((id.clone(), level_of(id)?)))
+ .collect())
+}
+
+/// 大纲级别 0–8 是第 1–9 级标题,9 是正文
+fn outline_level(outline: u8) -> Option<u8> {
+ (outline <= 8).then_some(outline + 1)
+}
+
+/// 内置标题样式的名字 `heading 1` … `heading 9`,大小写不论
+fn heading_number(name: &str) -> Option<u8> {
+ let name = name.trim().to_ascii_lowercase();
+ let n: u8 = name.strip_prefix("heading")?.trim().parse().ok()?;
+ (1..=9).contains(&n).then_some(n)
}
type GridRow = Vec<(String, usize, u32)>;
-pub(crate) fn docx_xml_to_text(xml: &str) -> anyhow::Result<String> {
+pub(crate) fn docx_xml_to_text(
+ xml: &str,
+ headings: &std::collections::HashMap<String, u8>,
+) -> anyhow::Result<String> {
let mut reader = Reader::from_str(xml);
let mut out = String::new();
let mut in_text = false;
@@ -153,6 +252,14 @@ pub(crate) fn docx_xml_to_text(xml: &str) -> anyhow::Result<String> {
let mut rows: Vec<GridRow> = Vec::new();
let mut row: GridRow = Vec::new();
let mut cell: Option<(String, usize, u32)> = None;
+ // 正文里的一段(不在表格里、不是文本框里套着的段)是不是标题。`paragraphs` 数套了几层
+ // w:p,`para_start` 是这一段在 out 里开始的位置。级别先看段落自己写的 w:outlineLvl,
+ // 再看它的样式;两样都只认第一次出现的,w:pPrChange 里记的是修订之前的样式,不算
+ let mut paragraphs = 0usize;
+ let mut para_start = 0usize;
+ let mut own_level: Option<Option<u8>> = None;
+ let mut style_level: Option<Option<u8>> = None;
+ let mut in_revision = false;
let attr = |e: &quick_xml::events::BytesStart<'_>, name: &str| -> Option<String> {
e.attributes()
.flatten()
@@ -161,6 +268,25 @@ pub(crate) fn docx_xml_to_text(xml: &str) -> anyhow::Result<String> {
};
loop {
match reader.read_event() {
+ Ok(Event::Start(e) | Event::Empty(e))
+ if matches!(e.name().as_ref(), "w:pStyle" | "w:outlineLvl") =>
+ {
+ if paragraphs == 1 && cell.is_none() && !in_revision {
+ if e.name().as_ref() == "w:pStyle" {
+ style_level.get_or_insert_with(|| {
+ attr(&e, "w:val").and_then(|id| headings.get(&id).copied())
+ });
+ } else {
+ own_level.get_or_insert_with(|| {
+ attr(&e, "w:val")
+ .and_then(|v| v.parse().ok())
+ .and_then(outline_level)
+ });
+ }
+ }
+ }
+ Ok(Event::Start(e)) if e.name().as_ref() == "w:pPrChange" => in_revision = true,
+ Ok(Event::End(e)) if e.name().as_ref() == "w:pPrChange" => in_revision = false,
Ok(Event::Start(e) | Event::Empty(e))
if matches!(e.name().as_ref(), "w:br" | "w:cr") =>
{
@@ -173,6 +299,14 @@ pub(crate) fn docx_xml_to_text(xml: &str) -> anyhow::Result<String> {
}
Ok(Event::Start(e)) => match e.name().as_ref() {
"w:t" => in_text = true,
+ "w:p" => {
+ paragraphs += 1;
+ if paragraphs == 1 && cell.is_none() {
+ para_start = out.len();
+ own_level = None;
+ style_level = None;
+ }
+ }
"w:tbl" => {
depth += 1;
if depth == 1 {
@@ -210,10 +344,31 @@ pub(crate) fn docx_xml_to_text(xml: &str) -> anyhow::Result<String> {
},
Ok(Event::End(e)) => match e.name().as_ref() {
"w:t" => in_text = false,
- "w:p" => match cell.as_mut() {
- Some(c) => c.0.push(' '),
- None => out.push('\n'),
- },
+ "w:p" => {
+ match cell.as_mut() {
+ Some(c) => c.0.push(' '),
+ None => {
+ // 标题写成一行 Markdown 标题:分块器靠它给每块开头补上所在的各级标题,
+ // 时间解释靠块里的标题行分节(0064 决定 1 按 cut 2 修订的那段)。没有
+ // 它,一份 Word 里各节的日期都算成第一节的
+ let level = own_level.unwrap_or(style_level.flatten());
+ if let Some(level) = level.filter(|_| paragraphs == 1) {
+ let title = out[para_start..]
+ .split_whitespace()
+ .collect::<Vec<_>>()
+ .join(" ");
+ if !title.is_empty() {
+ out.truncate(para_start);
+ out.push_str(&"#".repeat(usize::from(level.min(6))));
+ out.push(' ');
+ out.push_str(&title);
+ }
+ }
+ out.push('\n');
+ }
+ }
+ paragraphs = paragraphs.saturating_sub(1);
+ }
"w:tc" if depth == 1 => {
if let Some(c) = cell.take() {
row.push(c);
@@ -670,8 +825,10 @@ pub fn csv_text(bytes: &[u8], tsv: bool) -> anyhow::Result<String> {
// ---- 工具 ----
-fn read_zip_entry(bytes: &[u8], name: &str) -> anyhow::Result<String> {
- let mut archive = zip::ZipArchive::new(Cursor::new(bytes.to_vec()))?;
+fn read_zip_entry(
+ archive: &mut zip::ZipArchive<Cursor<&[u8]>>,
+ name: &str,
+) -> anyhow::Result<String> {
let mut entry = archive.by_name(name)?;
let mut content = String::new();
entry.read_to_string(&mut content)?;
@@ -729,7 +886,7 @@ mod tests {
/// docx 的表:跨列的列头按列拼,"$" 并回数字,表前后的段落照旧
#[test]
fn a_docx_table_is_rendered_under_its_headings() {
- let text = docx_xml_to_text(DOC).unwrap();
+ let text = docx_xml_to_text(DOC, &Default::default()).unwrap();
assert!(text.starts_with("Segment results\n"), "{text}");
assert!(
text.contains(
@@ -747,11 +904,43 @@ mod tests {
<w:tr><w:tc><w:p><w:r><w:t>Region</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>Notes</w:t></w:r></w:p></w:tc></w:tr>
<w:tr><w:tc><w:p><w:r><w:t>North</w:t></w:r></w:p></w:tc><w:tc><w:tbl><w:tr><w:tc><w:p><w:r><w:t>inner</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>7</w:t></w:r></w:p></w:tc></w:tr></w:tbl></w:tc></w:tr>
</w:tbl></w:body></w:document>"#;
- let text = docx_xml_to_text(xml).unwrap();
+ let text = docx_xml_to_text(xml, &Default::default()).unwrap();
assert_eq!(text.matches("| --- |").count(), 1, "{text}");
assert!(text.contains("| North | inner 7 |"), "{text}");
}
+ /// 标题样式按大纲级别认,其次按内置名字,再顺着 basedOn 找;样式 id 不作数
+ #[test]
+ fn heading_styles_are_known_by_outline_level_then_name_then_parent() {
+ let styles = r#"<w:styles xmlns:w="x">
+<w:style w:type="paragraph" w:default="1" w:styleId="a"><w:name w:val="Normal"/></w:style>
+<w:style w:type="paragraph" w:styleId="1"><w:name w:val="heading 1"/><w:basedOn w:val="a"/><w:pPr><w:keepNext/><w:outlineLvl w:val="0"/></w:pPr></w:style>
+<w:style w:type="paragraph" w:styleId="2"><w:name w:val="heading 2"/><w:basedOn w:val="a"/></w:style>
+<w:style w:type="paragraph" w:styleId="Heading3"><w:name w:val="Heading 3"/></w:style>
+<w:style w:type="paragraph" w:styleId="ReportSection"><w:name w:val="Report Section"/><w:basedOn w:val="2"/></w:style>
+<w:style w:type="paragraph" w:styleId="Outlined"><w:name w:val="Outlined"/><w:pPr><w:outlineLvl w:val="3"/></w:pPr></w:style>
+<w:style w:type="paragraph" w:styleId="Demoted"><w:name w:val="Demoted"/><w:basedOn w:val="1"/><w:pPr><w:outlineLvl w:val="9"/></w:pPr></w:style>
+<w:style w:type="paragraph" w:styleId="a3"><w:name w:val="Title"/><w:basedOn w:val="a"/></w:style>
+<w:style w:type="paragraph" w:styleId="TOC1"><w:name w:val="toc 1"/><w:basedOn w:val="a"/></w:style>
+<w:style w:type="character" w:styleId="10"><w:name w:val="Heading 1 Char"/><w:basedOn w:val="a0"/></w:style>
+<w:style w:type="paragraph" w:styleId="Loop1"><w:name w:val="Loop 1"/><w:basedOn w:val="Loop2"/></w:style>
+<w:style w:type="paragraph" w:styleId="Loop2"><w:name w:val="Loop 2"/><w:basedOn w:val="Loop1"/></w:style>
+</w:styles>"#;
+ let levels = docx_heading_styles(styles).unwrap();
+ let mut found: Vec<(&str, u8)> = levels.iter().map(|(id, l)| (id.as_str(), *l)).collect();
+ found.sort();
+ assert_eq!(
+ found,
+ [
+ ("1", 1),
+ ("2", 2),
+ ("Heading3", 3),
+ ("Outlined", 4),
+ ("ReportSection", 2)
+ ]
+ );
+ }
+
/// csv 的第一条记录是列头,哪怕列头是年份
#[test]
fn a_csv_is_a_table_whose_first_record_is_the_header() {
diff --git a/crates/utopia-ingest/tests/docx_headings.rs b/crates/utopia-ingest/tests/docx_headings.rs
new file mode 100644
index 00000000..2e0fd299
--- /dev/null
+++ b/crates/utopia-ingest/tests/docx_headings.rs
@@ -0,0 +1,118 @@
+//! Word 文档的标题读成标题。
+//!
+//! 照中文 Word 写出来的样子拼一份:样式 id 是 `1`、`2`(正文是 `a`),样式名是英文的
+//! `heading 1`,大纲级别写在样式里。一份里两份周报各有自己的日期——同一篇里分节,
+//! 是 0064 按节取日期要认的形状。
+
+use std::io::{Cursor, Write};
+
+const STYLES: &str = r#"<w:styles xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"><w:style w:type="paragraph" w:default="1" w:styleId="a"><w:name w:val="Normal"/><w:qFormat/></w:style><w:style w:type="paragraph" w:styleId="1"><w:name w:val="heading 1"/><w:basedOn w:val="a"/><w:next w:val="a"/><w:link w:val="10"/><w:uiPriority w:val="9"/><w:qFormat/><w:pPr><w:keepNext/><w:keepLines/><w:spacing w:before="340" w:after="330" w:line="578" w:lineRule="auto"/><w:outlineLvl w:val="0"/></w:pPr><w:rPr><w:b/><w:bCs/><w:kern w:val="44"/><w:sz w:val="44"/><w:szCs w:val="44"/></w:rPr></w:style><w:style w:type="paragraph" w:styleId="2"><w:name w:val="heading 2"/><w:basedOn w:val="a"/><w:next w:val="a"/><w:link w:val="20"/><w:uiPriority w:val="9"/><w:unhideWhenUsed/><w:qFormat/><w:pPr><w:keepNext/><w:keepLines/><w:spacing w:before="260" w:after="260" w:line="416" w:lineRule="auto"/><w:outlineLvl w:val="1"/></w:pPr></w:style><w:style w:type="paragraph" w:styleId="a3"><w:name w:val="Title"/><w:basedOn w:val="a"/><w:next w:val="a"/><w:qFormat/><w:pPr><w:jc w:val="center"/></w:pPr></w:style><w:style w:type="character" w:styleId="10"><w:name w:val="标题 1 字符"/><w:basedOn w:val="a0"/><w:link w:val="1"/></w:style></w:styles>"#;
+
+fn read(body: &str, styles: Option<&str>) -> String {
+ let mut parts = vec![
+ ("[Content_Types].xml", r#"<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"><Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/><Default Extension="xml" ContentType="application/xml"/><Override PartName="/word/document.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"/><Override PartName="/word/styles.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.styles+xml"/></Types>"#.to_string()),
+ ("_rels/.rels", r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="word/document.xml"/></Relationships>"#.to_string()),
+ ("word/_rels/document.xml.rels", r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/styles" Target="styles.xml"/></Relationships>"#.to_string()),
+ ("word/document.xml", format!(r#"<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"><w:body>{body}<w:sectPr/></w:body></w:document>"#)),
+ ];
+ if let Some(styles) = styles {
+ parts.push(("word/styles.xml", styles.to_string()));
+ }
+ let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new()));
+ for (part, xml) in parts {
+ zip.start_file(part, zip::write::SimpleFileOptions::default())
+ .unwrap();
+ zip.write_all(xml.as_bytes()).unwrap();
+ }
+ utopia_ingest::parse("周报.docx", &zip.finish().unwrap().into_inner())
+ .unwrap()
+ .text
+}
+
+fn styled(style: &str, runs: &str) -> String {
+ format!(r#"<w:p><w:pPr><w:pStyle w:val="{style}"/></w:pPr>{runs}</w:p>"#)
+}
+
+fn plain(text: &str) -> String {
+ format!("<w:p><w:r><w:t>{text}</w:t></w:r></w:p>")
+}
+
+fn run(text: &str) -> String {
+ format!("<w:r><w:t>{text}</w:t></w:r>")
+}
+
+fn reports() -> String {
+ [
+ styled("a3", &run("周报汇编")),
+ styled("1", &run("第一周周报(2024-03-01)")),
+ plain("上周销售额增长 5%。"),
+ // 标题里换了行:标题还是一行
+ styled(
+ "1",
+ &format!("{}<w:r><w:br/><w:t>(2024-03-08)</w:t></w:r>", run("第二周周报")),
+ ),
+ styled("2", &run("明细")),
+ plain("上周退货 3 单。"),
+ // 段落自己写的大纲级别,不靠样式
+ r#"<w:p><w:pPr><w:outlineLvl w:val="0"/></w:pPr><w:r><w:t>附录</w:t></w:r></w:p>"#.into(),
+ // 借了标题样式、又把大纲级别改回正文的一段
+ r#"<w:p><w:pPr><w:pStyle w:val="1"/><w:outlineLvl w:val="9"/></w:pPr><w:r><w:t>借了标题样式的正文</w:t></w:r></w:p>"#.into(),
+ // 修订:原来是标题,改成了正文;w:pPrChange 里记的是改之前的样式
+ r#"<w:p><w:pPr><w:pPrChange w:id="1" w:author="A" w:date="2024-03-09T00:00:00Z"><w:pPr><w:pStyle w:val="1"/></w:pPr></w:pPrChange></w:pPr><w:r><w:t>改回正文的一段</w:t></w:r></w:p>"#.into(),
+ format!(
+ "<w:tbl><w:tr><w:tc>{}</w:tc><w:tc>{}</w:tc></w:tr><w:tr><w:tc>{}</w:tc><w:tc>{}</w:tc></w:tr></w:tbl>",
+ plain("项目"),
+ plain("数量"),
+ styled("1", &run("格子里的标题样式")),
+ plain("3")
+ ),
+ ]
+ .concat()
+}
+
+#[test]
+fn a_paragraph_with_an_outline_level_is_a_heading() {
+ let text = read(&reports(), Some(STYLES));
+ assert!(
+ text.contains("# 第一周周报(2024-03-01)\n上周销售额增长 5%。\n"),
+ "{text}"
+ );
+ assert!(
+ text.contains("# 第二周周报 (2024-03-08)\n## 明细\n上周退货 3 单。\n"),
+ "{text}"
+ );
+ assert!(text.contains("# 附录\n"), "{text}");
+ for body in ["周报汇编", "借了标题样式的正文", "改回正文的一段"] {
+ assert!(text.lines().any(|line| line == body), "{body}: {text}");
+ }
+ assert!(text.contains("| 格子里的标题样式 | 3 |"), "{text}");
+}
+
+#[test]
+fn each_report_is_its_own_section_in_the_chunks() {
+ let text = read(&reports(), Some(STYLES));
+ let pieces = utopia_ingest::chunk_with_budget(&text, 40);
+ let chunk = |sentence: &str| {
+ pieces
+ .iter()
+ .find(|p| p.text.contains(sentence))
+ .map(|p| p.text[..p.text.find(sentence).unwrap()].to_string())
+ .unwrap_or_else(|| panic!("{sentence} is in no chunk: {pieces:#?}"))
+ };
+ // 分节看的是块里这句话前面的标题行(`headings_at` 读的就是它们)
+ let first = chunk("上周销售额增长");
+ assert!(first.contains("# 第一周周报(2024-03-01)"), "{first}");
+ let second = chunk("上周退货");
+ assert!(
+ second.contains("# 第二周周报 (2024-03-08)") && second.contains("## 明细"),
+ "{second}"
+ );
+ assert!(!second.contains("第一周周报"), "{second}");
+}
+
+#[test]
+fn without_a_style_sheet_only_a_paragraphs_own_outline_level_counts() {
+ let text = read(&reports(), None);
+ assert!(text.contains("# 附录\n"), "{text}");
+ assert!(!text.contains("# 第一周周报"), "{text}");
+}
diff --git a/docs/design/sources.md b/docs/design/sources.md
index 856d3ad9..274071d8 100644
--- a/docs/design/sources.md
+++ b/docs/design/sources.md
@@ -35,7 +35,10 @@ table travels with its caption and header and a continuation repeats both; a cap
every table that follows it; a row is never split; headings become a breadcrumb prefix and
`chunks.heading`; a thematic break is not a boundary and a table it split is joined back; no
overlap; chunk text is verbatim slices so a quote is found by offset; a document is re-chunked on
-its next reprocess [0039].
+its next reprocess [0039]. A DOCX paragraph outside a table is a heading when it has an outline
+level: its own `w:outlineLvl`, else its style's, else a style named `heading N`, else the style it
+is based on (style ids differ by the language of Word, the names do not); so a Word file's
+sections reach the chunks and the section dates of 0064.
**Tables** [#744, #750, prior-work item 26]. An HTML table is rendered from the DOM before the Markdown
converter sees it, by structure alone: cells hidden by style are skipped, spanning cells are laid on