Skip to content

World's first open-source enterprise world model.

A table keeps its headings, its sections and its caption (#744)

WaylandYangfc456184bb6b
diff --git a/Cargo.lock b/Cargo.lock
index 66de830c..e09af1e8 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5425,6 +5425,7 @@ dependencies = [
"calamine",
"chardetng",
"csv",
+ "dom_query",
"dom_smoothie",
"encoding_rs",
"flate2",
diff --git a/Cargo.toml b/Cargo.toml
index bb197b5d..950b5787 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -78,6 +78,7 @@ cron = "0.12"
infer = "0.22"
sha2 = "0.11"
dom_smoothie = "0.16"
+dom_query = "0.26"
htmd = "0.5"
pulldown-cmark = { version = "0.13", default-features = false }
chardetng = "1"
diff --git a/crates/utopia-extract/src/open.rs b/crates/utopia-extract/src/open.rs
index 895c0f12..f2b140cf 100644
--- a/crates/utopia-extract/src/open.rs
+++ b/crates/utopia-extract/src/open.rs
@@ -145,8 +145,14 @@ what is measured or stated (\"area\", \"was completed\", \"joined\", \"占地面
literal as written. Wording that names or describes something is not a value: that something \
goes in \"e\" and is linked by a statement with an object. \"1,200 beekeepers joined\" is the \
thing \"beekeepers\" with \"joined\" = \"1,200\". Write every figure the passage states. In a \
-table, a cell is a statement about its row's thing whose phrase is the column heading; leave \
-out cells whose column heading is not in the passage.\n\
+table, a cell is a statement about its row's thing whose phrase is the column heading. When \
+the column heading names one time (a date, a quarter, a period), it is when instead, the \
+statement is about the thing the table's caption names (the company of a financial statement) \
+and the phrase is the row label as written, path included (\"Operating expenses › Research and \
+development\"); a heading that names a comparison between periods (a change from the quarter \
+before, a change from a year ago) is not a time, it stays the phrase; a unit the caption or a \
+heading gives (\"In millions\") is a qualifier on each such statement. Leave out cells whose \
+column heading is not in the passage.\n\
A list is one statement per member, and so is a subject or object that joins several things \
(\"the city and the county funded the bridge\" is two statements).\n\
3. Nothing in a sentence is dropped. When more than two things take part, or the link carries \
diff --git a/crates/utopia-ingest/Cargo.toml b/crates/utopia-ingest/Cargo.toml
index 8bb639a0..12e97412 100644
--- a/crates/utopia-ingest/Cargo.toml
+++ b/crates/utopia-ingest/Cargo.toml
@@ -13,6 +13,7 @@ pdf-extract.workspace = true
calamine.workspace = true
htmd.workspace = true
dom_smoothie.workspace = true
+dom_query.workspace = true
thiserror.workspace = true
percent-encoding = "2.3.2"
chardetng.workspace = true
diff --git a/crates/utopia-ingest/src/chunker.rs b/crates/utopia-ingest/src/chunker.rs
index e6a58795..031e7c15 100644
--- a/crates/utopia-ingest/src/chunker.rs
+++ b/crates/utopia-ingest/src/chunker.rs
@@ -47,6 +47,8 @@ pub const BUDGET_TOKENS: usize = 300;
/// 说明句最长多少字节还算说明句:表上面那一段要短、或者以冒号结尾,
/// 才当成表的一部分带着走;一整段分析不是说明句
const CAPTION_MAX_BYTES: usize = 200;
+/// 表前最多几段短说明一起当说明句
+const CAPTION_RUN_MAX: usize = 5;
/// 不到这么多 token 的一段(页码、脚注标记)不单独成块,并进上一块
const TINY_TOKENS: usize = 8;
@@ -100,11 +102,24 @@ fn units(text: &str, blocks: Vec<Block>) -> Vec<Unit> {
// 提名人,结果如下:」后面是十张表,从前只有第一张带着它;黄仁勋
// 那张在另一块里,四个票数都对,却没有一句话说这是在投什么,
// 「当选董事」那条边就没了。一串到别的东西(段落、标题)出现为止。
- let caption_like = b.range.len() <= CAPTION_MAX_BYTES
- || text[b.range.clone()].trim_end().ends_with(':');
- let mut j = i + 1;
+ //
+ // - 表前**连续几段**短说明一起算:财报的「NVIDIA CORPORATION」「CONDENSED
+ // CONSOLIDATED STATEMENTS OF INCOME」「(In millions)」「(Unaudited)」是四个
+ // 段落,只带最后一段的话,模型看到的表不知道是哪张报表、什么单位。
+ // 一串最多五段,中间不隔别的东西
+ let caption_like = |blk: &Block| {
+ matches!(blk.kind, Kind::Paragraph)
+ && (blk.range.len() <= CAPTION_MAX_BYTES
+ || text[blk.range.clone()].trim_end().ends_with(':'))
+ };
+ let mut k = i + 1;
+ while k - i < CAPTION_RUN_MAX && blocks.get(k).is_some_and(&caption_like) {
+ k += 1;
+ }
+ let caption = b.range.start..blocks[k - 1].range.end;
+ let mut j = k;
let mut attached = 0usize;
- if caption_like {
+ if caption_like(b) {
loop {
while matches!(
blocks.get(j),
@@ -123,7 +138,7 @@ fn units(text: &str, blocks: Vec<Block>) -> Vec<Unit> {
break;
};
out.push(Unit::Table {
- caption: Some(b.range.clone()),
+ caption: Some(caption.clone()),
head: head.clone(),
rows: rows.clone(),
});
@@ -132,6 +147,7 @@ fn units(text: &str, blocks: Vec<Block>) -> Vec<Unit> {
}
}
if attached == 0 {
+ // 不是说明句:这一段照常;后面那几段下一轮各自再看
out.push(Unit::Text(b.range.clone()));
} else {
i = j - 1;
@@ -769,6 +785,45 @@ The tenant pays rent monthly.
}
}
+ /// 财报的形状:公司名、报表名、单位、「未经审计」四段短说明,然后是表。
+ /// 四段都是说明句,切开的每一块都带着全部四段。
+ #[test]
+ fn a_run_of_short_paragraphs_before_a_table_is_its_caption() {
+ let text = format!(
+ "Prose that is long enough not to be a caption, going on about the quarter and the outlook and the products and the customers.\n\n\
+ NVIDIA CORPORATION\n\nCONDENSED CONSOLIDATED STATEMENTS OF INCOME\n\n(In millions)\n\n(Unaudited)\n\n{}",
+ table(30)
+ );
+ let pieces = chunk_with_budget(&text, 120);
+ let with_table: Vec<&ChunkPiece> = pieces
+ .iter()
+ .filter(|p| p.text.contains("| --- |"))
+ .collect();
+ assert!(with_table.len() >= 2, "表该被切成几块: {}", pieces.len());
+ for p in &with_table {
+ assert!(
+ p.text.contains("NVIDIA CORPORATION"),
+ "每块都带公司名: {}",
+ p.text
+ );
+ assert!(
+ p.text.contains("STATEMENTS OF INCOME"),
+ "每块都带报表名: {}",
+ p.text
+ );
+ assert!(
+ p.text.contains("(Unaudited)"),
+ "每块都带最后一段: {}",
+ p.text
+ );
+ assert!(
+ !p.text.contains("Prose that is long"),
+ "长段落不是说明句: {}",
+ p.text
+ );
+ }
+ }
+
/// 十位董事的形状:一句说明、一串表。每张表不论落在哪一块都看得见那句说明;
/// 同一块里的几张表,说明只念一遍。
#[test]
diff --git a/crates/utopia-ingest/src/html.rs b/crates/utopia-ingest/src/html.rs
index 2a9895f0..a6c42d49 100644
--- a/crates/utopia-ingest/src/html.rs
+++ b/crates/utopia-ingest/src/html.rs
@@ -488,7 +488,10 @@ fn prune_empty_table_columns(markdown: &str) -> String {
}
fn markdown_from_html(html: &str) -> Result<String, HtmlError> {
- let html = &promote_first_row_headers(html);
+ // 最外层的表先从 DOM 渲染成带真表头的 Markdown(见 `table`),htmd 只看到占位段落;
+ // 套着的表还走老路,所以第一行提表头的补丁留着给它们
+ let (html, tables) = crate::table::lift_tables(html);
+ let html = &promote_first_row_headers(&html);
let markdown = htmd::HtmlToMarkdown::builder()
.skip_tags(vec![
"script", "style", "iframe", "object", "embed", "img", "svg", "math",
@@ -505,6 +508,7 @@ fn markdown_from_html(html: &str) -> Result<String, HtmlError> {
{
return Err(HtmlError::Interstitial);
}
+ let markdown = crate::table::restore_tables(&markdown, &tables);
normalize_markdown(&prune_empty_table_columns(&markdown))
}
diff --git a/crates/utopia-ingest/src/lib.rs b/crates/utopia-ingest/src/lib.rs
index 8e940fcd..94a76e44 100644
--- a/crates/utopia-ingest/src/lib.rs
+++ b/crates/utopia-ingest/src/lib.rs
@@ -169,6 +169,7 @@ pub mod ontology_rdf;
mod parsers;
pub mod provenance;
mod reading;
+mod table;
pub mod transcript;
pub use chunker::{chunk_segments, chunk_text, chunk_with_budget, ChunkPiece, BUDGET_TOKENS};
diff --git a/crates/utopia-ingest/src/table.rs b/crates/utopia-ingest/src/table.rs
new file mode 100644
index 00000000..97f82fd7
--- /dev/null
+++ b/crates/utopia-ingest/src/table.rs
@@ -0,0 +1,681 @@
+//! 表格从 DOM 直接渲染成带真表头的 Markdown 表(0039/0040 那一层的活)。
+//!
+//! 为什么不交给 htmd:SEC/XBRL 报表的表一个 `<th>` 都没有。标题行(公司名、报表名、
+//! 单位)是 colspan 跨全宽的普通行,列头("July 26," / "2026")拆在两行里,每个数字前后
+//! 各有一格 "$" 或 ")",列与列之间是空的占位格。此前的补丁把第一行提成表头,于是被切开
+//! 的每一块表头都是「| NVIDIA CORPORATION | | |」,真列头一块都没带;模型面对
+//! 「Accounts receivable, net | 63,059 | 38,466」只能写出「短语 = 数字」或者不写。财报
+//! 长文在 #743 下 56 条错说法里 38 条、64 条读不通里 43 条出自这里。
+//!
+//! 这里只看结构,不认任何词:跨列格、空列、只含符号的格、只有一格的行、行标签的左内
+//! 边距、格子里有没有数字和字母。做四件事:标题行提成表前的说明句;多行列头按列拼成
+//! 一行;只有标签没有值的行当成小节,折进后面每行的标签里(「Current assets ›
+//! Accounts receivable, net」);"$"、")" 这类符号格并回相邻的数字格。渲染出的表交给
+//! `blocks` / `chunker`,它们本来就让说明句和表头跟着每一块走。
+//!
+//! 只处理最外层且不含内层表的表;套着的表留给 htmd 原路。
+
+use dom_query::{Document, Selection};
+
+const PLACEHOLDER: &str = "UTOPIATABLE";
+const PATH_SEP: &str = " › ";
+
+/// 一格:文字、跨了几列、左内边距(pt,取整)、在网格里的起始列。
+#[derive(Clone, Debug, Default)]
+struct Cell {
+ text: String,
+ span: usize,
+ pad: u32,
+ col: usize,
+}
+
+#[derive(Clone, Debug)]
+struct Row {
+ cells: Vec<Cell>,
+ /// 网格总宽(含跨列)
+ width: usize,
+}
+
+impl Row {
+ /// 覆盖第 k 列的那个非空格
+ fn covering(&self, k: usize) -> Option<&Cell> {
+ self.cells
+ .iter()
+ .find(|c| !c.text.is_empty() && c.col <= k && k < c.col + c.span)
+ }
+}
+
+#[derive(Debug, PartialEq, Eq)]
+enum Kind {
+ Skip,
+ Caption,
+ Header,
+ Section,
+ Data,
+}
+
+/// 把最外层且不含内层表的表换成占位段落;返回改过的 HTML 和每个占位对应的 Markdown。
+pub fn lift_tables(html: &str) -> (String, Vec<String>) {
+ let doc = Document::from(html);
+ let mut rendered: Vec<String> = Vec::new();
+ let mut targets: Vec<Selection<'_>> = Vec::new();
+ // 上一张表的列头:分页符把一张报表劈成两个 <table> 时,后半张没有表头,
+ // 列数又一样,就是同一张表的续表,接着用前一张的列头
+ let mut last_headers: Vec<String> = Vec::new();
+ for tbl in doc.select("table").iter() {
+ let nested_in = tbl.nodes().first().is_some_and(|n| {
+ n.ancestors(None)
+ .iter()
+ .any(|a| a.node_name().as_deref() == Some("table"))
+ });
+ if nested_in || tbl.select("table").exists() {
+ continue;
+ }
+ if let Some((md, headers)) = render_table(&tbl, &last_headers) {
+ rendered.push(md);
+ targets.push(tbl);
+ last_headers = headers;
+ }
+ }
+ if rendered.is_empty() {
+ return (html.to_string(), rendered);
+ }
+ for (i, tbl) in targets.iter().enumerate() {
+ tbl.replace_with_html(format!("<p>{PLACEHOLDER}{i}</p>"));
+ }
+ (doc.html().to_string(), rendered)
+}
+
+/// 把 htmd 转出的 Markdown 里的占位行换回渲染好的表。
+pub fn restore_tables(markdown: &str, tables: &[String]) -> String {
+ if tables.is_empty() {
+ return markdown.to_string();
+ }
+ let extra: usize = tables.iter().map(String::len).sum();
+ let mut out = String::with_capacity(markdown.len() + extra);
+ for line in markdown.lines() {
+ let hit = line
+ .trim()
+ .strip_prefix(PLACEHOLDER)
+ .and_then(|rest| rest.parse::<usize>().ok())
+ .and_then(|i| tables.get(i));
+ match hit {
+ Some(md) => {
+ out.push('\n');
+ out.push_str(md);
+ out.push('\n');
+ }
+ None => out.push_str(line),
+ }
+ out.push('\n');
+ }
+ out
+}
+
+fn hidden(style: &str) -> bool {
+ let s: String = style.chars().filter(|c| !c.is_whitespace()).collect();
+ s.to_ascii_lowercase().contains("display:none")
+}
+
+/// 左内边距:`padding-left: 12pt`、`text-indent: 6pt`,或 `padding: a b c d` 的第四个值。
+fn padding_left(style: &str) -> u32 {
+ let lower = style.to_ascii_lowercase();
+ let number_at = |s: &str| -> Option<f32> {
+ let s = s.trim_start();
+ let end = s
+ .find(|c: char| !(c.is_ascii_digit() || c == '.'))
+ .unwrap_or(s.len());
+ s[..end].parse::<f32>().ok()
+ };
+ for key in ["padding-left:", "text-indent:"] {
+ if let Some(i) = lower.find(key) {
+ if let Some(v) = number_at(&lower[i + key.len()..]) {
+ return v.round() as u32;
+ }
+ }
+ }
+ if let Some(i) = lower.find("padding:") {
+ let rest = lower[i + "padding:".len()..]
+ .split(';')
+ .next()
+ .unwrap_or("");
+ let parts: Vec<&str> = rest.split_whitespace().collect();
+ let left = match parts.len() {
+ 4 => parts.get(3),
+ 2 | 3 => parts.get(1),
+ 1 => parts.first(),
+ _ => None,
+ };
+ if let Some(v) = left.and_then(|p| number_at(p)) {
+ return v.round() as u32;
+ }
+ }
+ 0
+}
+
+fn clean(text: &str) -> String {
+ text.replace('\u{a0}', " ")
+ .split_whitespace()
+ .collect::<Vec<_>>()
+ .join(" ")
+}
+
+fn attr_usize(cell: &Selection<'_>, name: &str) -> usize {
+ cell.attr(name)
+ .and_then(|v| v.trim().parse::<usize>().ok())
+ .unwrap_or(1)
+ .max(1)
+}
+
+fn is_numeric(text: &str) -> bool {
+ text.chars().any(|c| c.is_ascii_digit()) && !text.chars().any(char::is_alphabetic)
+}
+
+/// 该并回邻格的符号格:货币符号和左括号贴到后一格前面,右括号和百分号贴到前一格后面。
+/// 破折号那类「空值」占位符不算——它自己就是这一格的内容,并过去会把整行的值挪位
+fn merge_direction(text: &str) -> Option<bool> {
+ let mut chars = text.chars();
+ let first = chars.next()?;
+ if chars.next().is_some() {
+ return None;
+ }
+ if first == '(' || is_currency(first) {
+ Some(true)
+ } else if first == ')' || first == '%' {
+ Some(false)
+ } else {
+ None
+ }
+}
+
+fn is_currency(c: char) -> bool {
+ matches!(c, '$' | '€' | '£' | '¥' | '₩' | '₹' | '¢' | '¥')
+}
+
+/// 读出网格:按 `tr` 逐行,跨行的格顺着往下带。
+fn grid(tbl: &Selection<'_>) -> Vec<Row> {
+ let mut rows: Vec<Row> = Vec::new();
+ // (列, 格, 还剩几行)
+ let mut carry: Vec<(usize, Cell, usize)> = Vec::new();
+ for tr in tbl.select("tr").iter() {
+ let mut cells: Vec<Cell> = Vec::new();
+ let mut col = 0usize;
+ for cell in tr.select("td, th").iter() {
+ let style = cell
+ .attr("style")
+ .map(|s| s.to_string())
+ .unwrap_or_default();
+ if hidden(&style) {
+ continue;
+ }
+ place_carried(&mut col, &mut cells, &mut carry);
+ let span = attr_usize(&cell, "colspan");
+ let rowspan = attr_usize(&cell, "rowspan");
+ let c = Cell {
+ text: clean(&cell.text()),
+ span,
+ pad: padding_left(&style),
+ col,
+ };
+ if rowspan > 1 {
+ carry.push((col, c.clone(), rowspan - 1));
+ }
+ col += span;
+ cells.push(c);
+ }
+ place_carried(&mut col, &mut cells, &mut carry);
+ rows.push(Row { cells, width: col });
+ }
+ rows
+}
+
+fn place_carried(col: &mut usize, cells: &mut Vec<Cell>, carry: &mut Vec<(usize, Cell, usize)>) {
+ while let Some(pos) = carry.iter().position(|(c, _, _)| *c == *col) {
+ let (_, mut cell, left) = carry[pos].clone();
+ cell.col = *col;
+ *col += cell.span;
+ cells.push(cell);
+ if left <= 1 {
+ carry.remove(pos);
+ } else {
+ carry[pos].2 = left - 1;
+ }
+ }
+}
+
+/// 一行是什么:只看非空格的个数、位置、跨度,和格子里有没有数字。
+fn kind(row: &Row, headers_seen: bool, data_seen: bool, table_width: usize) -> Kind {
+ let nonempty: Vec<&Cell> = row.cells.iter().filter(|c| !c.text.is_empty()).collect();
+ let Some(first) = nonempty.first() else {
+ return Kind::Skip;
+ };
+ if nonempty.len() == 1 {
+ let wide = first.span >= 2 && first.span * 2 >= table_width.max(2);
+ if is_numeric(&first.text) {
+ return if data_seen { Kind::Data } else { Kind::Header };
+ }
+ if !headers_seen && !data_seen {
+ // 表头和数据都还没来:靠左或跨宽的一格文字是标题,不靠左的是一根列头
+ return if wide || first.col == 0 {
+ Kind::Caption
+ } else {
+ Kind::Header
+ };
+ }
+ return Kind::Section;
+ }
+ let label = !is_numeric(&first.text);
+ let values_after = nonempty[1..].iter().any(|c| is_numeric(&c.text));
+ // 有标签跟着数字的是数据行;最左一格就有内容的整排数字也是(年份列的表)。
+ // 最左几格空着、全是数字的一排("2026 | 2026")是列头的第二行
+ if values_after && (label || first.col == 0 || data_seen) {
+ return Kind::Data;
+ }
+ if !data_seen {
+ return Kind::Header;
+ }
+ if label {
+ // 数据行之后又一排没有数字的格:当成一条小节
+ return Kind::Section;
+ }
+ Kind::Data
+}
+
+/// 把只含符号的格并回邻格:"$" 和 "(" 贴到后一个非空格前面,")" 和 "%" 贴到前一个后面。
+fn merge_symbols(row: &mut Row) {
+ let n = row.cells.len();
+ for i in 0..n {
+ let Some(opening) = merge_direction(&row.cells[i].text) else {
+ continue;
+ };
+ let sym = row.cells[i].text.clone();
+ let target = if opening {
+ (i + 1..n).find(|&j| !row.cells[j].text.is_empty())
+ } else {
+ (0..i).rev().find(|&j| !row.cells[j].text.is_empty())
+ };
+ let Some(j) = target else { continue };
+ if opening {
+ row.cells[j].text = format!("{sym}{}", row.cells[j].text);
+ } else {
+ row.cells[j].text = format!("{}{sym}", row.cells[j].text);
+ }
+ row.cells[i].text.clear();
+ }
+}
+
+fn escape(text: &str) -> String {
+ text.replace('|', "\\|")
+}
+
+/// 缩进深度:先看标签格前面有几根空列(Workiva 报表用空格子缩进),再看左内边距
+/// (另一些用 padding-left)。两者合成一个数,列在前、内边距在后
+fn depth_of(cell: &Cell) -> u32 {
+ cell.col as u32 * 1000 + cell.pad
+}
+
+/// 一张表 → 说明句(可选)+ 带真表头的 Markdown 表,连同它用的列头(给续表接着用)。
+/// 表里什么都没有时返回 None。
+fn render_table(tbl: &Selection<'_>, inherited: &[String]) -> Option<(String, Vec<String>)> {
+ let mut rows = grid(tbl);
+ let width = rows.iter().map(|r| r.width).max().unwrap_or(0);
+ if width == 0 {
+ return None;
+ }
+ let mut captions: Vec<String> = Vec::new();
+ let mut headers: Vec<Row> = Vec::new();
+ // (小节深度, 小节名, 已经见过比它深的行)。
+ // 一个小节被同深度的数据行结束,只在它已经有过更深的子行之后:资产负债表里
+ // 「Current assets:」的子行缩进了,同深度的「Property and equipment」是下一项;
+ // 现金流量表里「Cash flows from operating activities:」的子行没缩进,同深度的
+ // 「Net income」就是它的第一行
+ let mut sections: Vec<(u32, String, bool)> = Vec::new();
+ // (标签路径, 该行)
+ let mut data: Vec<(String, Row)> = Vec::new();
+ let (mut headers_seen, mut data_seen) = (false, false);
+ for row in rows.iter_mut() {
+ match kind(row, headers_seen, data_seen, width) {
+ Kind::Skip => {}
+ Kind::Caption => {
+ if let Some(c) = row.cells.iter().find(|c| !c.text.is_empty()) {
+ captions.push(c.text.clone());
+ }
+ }
+ Kind::Header => {
+ merge_symbols(row);
+ headers.push(row.clone());
+ headers_seen = true;
+ }
+ Kind::Section => {
+ let Some(cell) = row.cells.iter().find(|c| !c.text.is_empty()) else {
+ continue;
+ };
+ let depth = depth_of(cell);
+ while sections.last().is_some_and(|(d, _, _)| *d >= depth) {
+ sections.pop();
+ }
+ sections.push((
+ depth,
+ cell.text.trim_end_matches([':', ':']).to_string(),
+ false,
+ ));
+ }
+ Kind::Data => {
+ merge_symbols(row);
+ let label_cell = row.cells.iter().find(|c| !c.text.is_empty());
+ let depth = label_cell.map(depth_of).unwrap_or(0);
+ while sections
+ .last()
+ .is_some_and(|(d, _, deeper)| *d > depth || (*d == depth && *deeper))
+ {
+ sections.pop();
+ }
+ if let Some(top) = sections.last_mut() {
+ if top.0 < depth {
+ top.2 = true;
+ }
+ }
+ let path: Vec<&str> = sections.iter().map(|(_, s, _)| s.as_str()).collect();
+ data.push((path.join(PATH_SEP), row.clone()));
+ data_seen = true;
+ }
+ }
+ }
+ if data.is_empty() && headers.is_empty() {
+ return None;
+ }
+
+ // 标签列 = 第一根出现过数字的列左边的所有列;一个数字都没有的表,标签列只有最左那格。
+ // 值列 = 标签列右边、数据行里出现过内容的列(表头有字但整列没数据的列不算)
+ let label_end: usize = (0..width)
+ .find(|&k| {
+ data.iter()
+ .any(|(_, r)| r.covering(k).is_some_and(|c| is_numeric(&c.text)))
+ })
+ .or_else(|| {
+ data.iter()
+ .filter_map(|(_, r)| {
+ r.cells
+ .iter()
+ .find(|c| !c.text.is_empty())
+ .map(|c| c.col + c.span)
+ })
+ .min()
+ })
+ .or_else(|| {
+ headers
+ .iter()
+ .filter_map(|h| h.cells.iter().find(|c| !c.text.is_empty()).map(|c| c.col))
+ .min()
+ })
+ .unwrap_or(0);
+ let has_content = |k: usize| -> bool {
+ if data.is_empty() {
+ headers.iter().any(|h| h.covering(k).is_some())
+ } else {
+ data.iter().any(|(_, r)| r.covering(k).is_some())
+ }
+ };
+ let value_cols: Vec<usize> = (label_end..width).filter(|&k| has_content(k)).collect();
+ let value_cols: Vec<Vec<usize>> = collapse_spans(value_cols, &headers, &data);
+ // 一根值列是一组网格列:取这一行落在这组里的第一个非空格
+ let at = |r: &Row, group: &[usize]| -> String {
+ group
+ .iter()
+ .find_map(|&k| r.covering(k))
+ .map(|c| c.text.clone())
+ .unwrap_or_default()
+ };
+ // 一行的标签:标签列里的格子按出现顺序拼起来
+ let own_label = |r: &Row| -> String {
+ let mut parts: Vec<&str> = Vec::new();
+ for c in r
+ .cells
+ .iter()
+ .filter(|c| !c.text.is_empty() && c.col < label_end)
+ {
+ if parts.last() != Some(&c.text.as_str()) {
+ parts.push(c.text.as_str());
+ }
+ }
+ parts.join(" ")
+ };
+ let column_header = |group: &[usize]| -> String {
+ let mut parts: Vec<String> = Vec::new();
+ for h in &headers {
+ let t = at(h, group);
+ if !t.is_empty() && parts.last() != Some(&t) {
+ parts.push(t);
+ }
+ }
+ parts.join(" ")
+ };
+ let mut header_texts: Vec<String> = value_cols.iter().map(|g| column_header(g)).collect();
+ if headers.is_empty() && !data.is_empty() && inherited.len() == header_texts.len() {
+ header_texts = inherited.to_vec();
+ }
+ // 标签列的表头:只算整格都落在标签列里的表头格,跨进值列的是值列的头
+ let label_header: String = {
+ let mut parts: Vec<String> = Vec::new();
+ for h in &headers {
+ for c in h
+ .cells
+ .iter()
+ .filter(|c| !c.text.is_empty() && c.col + c.span <= label_end)
+ {
+ if parts.last() != Some(&c.text) {
+ parts.push(c.text.clone());
+ }
+ }
+ }
+ parts.join(" ")
+ };
+
+ let mut lines: Vec<String> = Vec::new();
+ if !captions.is_empty() {
+ let mut caption = captions.join(" · ");
+ if !caption.ends_with(':') && !caption.ends_with(':') {
+ caption.push(':');
+ }
+ lines.push(caption);
+ lines.push(String::new());
+ }
+ // 最左一列本身就是数字(年份列的表):没有标签列,行就是它的值
+ let has_label_col = label_end > 0;
+ let ncols = usize::from(has_label_col) + value_cols.len().max(1);
+ let header_cells: Vec<String> = has_label_col
+ .then(|| escape(&label_header))
+ .into_iter()
+ .chain(header_texts.iter().map(|h| escape(h)))
+ .chain(std::iter::repeat(String::new()))
+ .take(ncols)
+ .collect();
+ lines.push(format!("| {} |", header_cells.join(" | ")));
+ lines.push(format!("|{}", " --- |".repeat(ncols)));
+ for (sections, r) in &data {
+ let mut cells: Vec<String> = Vec::new();
+ if has_label_col {
+ let own = own_label(r);
+ let label = match (sections.is_empty(), own.is_empty()) {
+ (true, _) => own,
+ (false, true) => sections.clone(),
+ (false, false) => format!("{sections}{PATH_SEP}{own}"),
+ };
+ cells.push(escape(&label));
+ }
+ cells.extend(value_cols.iter().map(|g| escape(&at(r, g))));
+ while cells.len() < ncols {
+ cells.push(String::new());
+ }
+ lines.push(format!("| {} |", cells.join(" | ")));
+ }
+ Some((lines.join("\n"), header_texts))
+}
+
+/// 跨列的格会让同一个值落在好几根网格列上,"$" 又常常单独占一格:相邻两列在每一行
+/// 上要么由同一格覆盖、要么至少一边是空的,就是同一根值列,并成一组。
+fn collapse_spans(cols: Vec<usize>, headers: &[Row], data: &[(String, Row)]) -> Vec<Vec<usize>> {
+ let mut out: Vec<Vec<usize>> = Vec::new();
+ for k in cols {
+ let same_as_prev = out.last().and_then(|g| g.last().copied()).is_some_and(|p| {
+ headers.iter().chain(data.iter().map(|(_, r)| r)).all(|r| {
+ match (r.covering(p), r.covering(k)) {
+ (Some(a), Some(b)) => a.col == b.col,
+ _ => true,
+ }
+ })
+ });
+ match out.last_mut() {
+ Some(g) if same_as_prev => g.push(k),
+ _ => out.push(vec![k]),
+ }
+ }
+ out
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn render(html: &str) -> String {
+ let (_, tables) = lift_tables(html);
+ tables.join("\n=====\n")
+ }
+
+ #[test]
+ fn a_filing_table_gets_its_real_headers_and_its_section_path() {
+ let html = r#"<table>
+ <tr><td style="width:0.1%"></td><td style="width:40%"></td><td style="width:0.1%"></td><td style="width:1%"></td><td style="width:20%"></td><td style="width:1%"></td><td style="width:1%"></td><td style="width:20%"></td><td style="width:0.1%"></td><td colspan="3" style="display:none"></td></tr>
+ <tr><td colspan="9" style="text-align:center">NVIDIA CORPORATION</td><td colspan="3" style="display:none"></td></tr>
+ <tr><td colspan="9" style="text-align:center">CONDENSED CONSOLIDATED BALANCE SHEETS</td></tr>
+ <tr><td colspan="9" style="text-align:center">(In millions)</td></tr>
+ <tr><td colspan="3"></td><td colspan="3" style="text-align:center">July 26,</td><td colspan="3" style="text-align:center">January 25,</td></tr>
+ <tr><td colspan="3"></td><td colspan="3" style="text-align:center">2026</td><td colspan="3" style="text-align:center">2026</td></tr>
+ <tr><td colspan="3" style="padding:2pt 1pt 2pt 6pt">Current assets:</td><td colspan="3"></td><td colspan="3"></td></tr>
+ <tr><td colspan="3" style="padding:2pt 1pt 2pt 12pt">Accounts receivable, net</td><td>$</td><td>63,059</td><td></td><td>$</td><td>38,466</td><td></td></tr>
+ <tr><td colspan="3" style="padding:2pt 1pt 2pt 12pt">Inventories</td><td></td><td>31,575</td><td></td><td></td><td>21,403</td><td></td></tr>
+ <tr><td colspan="3" style="padding:2pt 1pt 2pt 6pt">Total current assets</td><td></td><td>197,412</td><td></td><td></td><td>125,605</td><td></td></tr>
+ </table>"#;
+ let md = render(html);
+ assert!(
+ md.starts_with(
+ "NVIDIA CORPORATION · CONDENSED CONSOLIDATED BALANCE SHEETS · (In millions):\n"
+ ),
+ "标题行提成表前的说明句: {md}"
+ );
+ assert!(
+ md.contains("| | July 26, 2026 | January 25, 2026 |"),
+ "两行列头拼成一行: {md}"
+ );
+ assert!(
+ md.contains("| Current assets › Accounts receivable, net | $63,059 | $38,466 |"),
+ "小节折进标签、美元符并回数字: {md}"
+ );
+ assert!(
+ md.contains("| Current assets › Inventories | 31,575 | 21,403 |"),
+ "{md}"
+ );
+ assert!(
+ md.contains("| Total current assets | 197,412 | 125,605 |"),
+ "合计行回到小节外: {md}"
+ );
+ assert!(!md.contains("Current assets:"), "小节行自己不再出现: {md}");
+ }
+
+ #[test]
+ fn a_lone_label_before_everything_is_the_caption_and_rows_keep_their_values() {
+ let html = "<table><tr><td colspan=\"2\">a. Tench Coxe</td></tr>\
+ <tr><td>Number of shares For</td><td>15,411,252,412</td></tr>\
+ <tr><td>Number of shares Against</td><td>1,399,727</td></tr></table>";
+ let md = render(html);
+ assert!(md.starts_with("a. Tench Coxe:\n"), "{md}");
+ assert!(
+ md.contains("| Number of shares For | 15,411,252,412 |"),
+ "{md}"
+ );
+ assert!(
+ md.contains("| Number of shares Against | 1,399,727 |"),
+ "{md}"
+ );
+ }
+
+ #[test]
+ fn a_first_row_of_words_is_the_header_when_the_table_has_none() {
+ let html = "<table><tr><td>Revenue</td><td>Q2 FY27</td><td>Q1 FY27</td></tr>\
+ <tr><td>total</td><td>$96,221</td><td>$81,615</td></tr></table>";
+ let md = render(html);
+ assert!(md.contains("| Revenue | Q2 FY27 | Q1 FY27 |"), "{md}");
+ assert!(md.contains("| total | $96,221 | $81,615 |"), "{md}");
+ }
+
+ #[test]
+ fn hidden_cells_and_empty_columns_do_not_survive() {
+ let html = "<table><tr><td>Margin</td><td style=\"display:none\">ghost</td><td></td><td>75.0</td><td></td></tr></table>";
+ let md = render(html);
+ assert!(!md.contains("ghost"), "{md}");
+ assert!(md.contains("| Margin | 75.0 |"), "{md}");
+ }
+
+ #[test]
+ fn a_closing_paren_in_its_own_cell_rejoins_the_number() {
+ let html =
+ "<table><tr><td>Other</td><td>(5,497</td><td>)</td><td>387</td><td></td></tr></table>";
+ let md = render(html);
+ assert!(md.contains("| Other | (5,497) | 387 |"), "{md}");
+ }
+
+ #[test]
+ fn a_dash_that_means_nothing_stays_in_its_own_cell() {
+ let html =
+ "<table><tr><td>Other</td><td>112</td><td>—</td><td>31</td><td>—</td></tr></table>";
+ let md = render(html);
+ assert!(md.contains("| Other | 112 | — | 31 | — |"), "{md}");
+ }
+
+ #[test]
+ fn a_headerless_table_after_a_page_break_takes_the_headers_before_it() {
+ let html = "<table>\
+ <tr><td></td><td>Q2 FY27</td><td>Q1 FY27</td></tr>\
+ <tr><td>Revenue</td><td>96,221</td><td>81,615</td></tr></table>\
+ <hr>\
+ <table><tr><td colspan=\"3\">Cash flows from financing activities:</td></tr>\
+ <tr><td>Dividends paid</td><td>(6,047)</td><td>(244)</td></tr></table>";
+ let (_, tables) = lift_tables(html);
+ assert_eq!(tables.len(), 2, "{tables:?}");
+ assert!(
+ tables[1].starts_with("Cash flows from financing activities:\n"),
+ "{}",
+ tables[1]
+ );
+ assert!(
+ tables[1].contains("| | Q2 FY27 | Q1 FY27 |"),
+ "续表接着用前一张的列头: {}",
+ tables[1]
+ );
+ assert!(
+ tables[1].contains("| Dividends paid | (6,047) | (244) |"),
+ "{}",
+ tables[1]
+ );
+ }
+
+ #[test]
+ fn nested_tables_are_left_to_the_markdown_converter() {
+ let html =
+ "<table><tr><td><table><tr><td>inner</td><td>1</td></tr></table></td></tr></table>";
+ let (out, tables) = lift_tables(html);
+ assert!(tables.is_empty(), "{tables:?}");
+ assert_eq!(out, html);
+ }
+
+ #[test]
+ fn placeholders_come_back_as_tables() {
+ let md = "before\n\nUTOPIATABLE0\n\nafter";
+ let table = "cap:\n\n| | a |\n| --- | --- |\n| x | 1 |".to_string();
+ let out = restore_tables(md, &[table]);
+ assert!(
+ out.contains("before\n\n\ncap:\n\n| | a |\n| --- | --- |\n| x | 1 |\n\n\nafter"),
+ "{out}"
+ );
+ }
+}
diff --git a/crates/utopia-server/src/extraction_open.rs b/crates/utopia-server/src/extraction_open.rs
index a5b94f8c..59dabb46 100644
--- a/crates/utopia-server/src/extraction_open.rs
+++ b/crates/utopia-server/src/extraction_open.rs
@@ -61,11 +61,18 @@ fn locate(hay: &str, needle: &str) -> Option<(i32, i32)> {
/// 整块里搜得到不等于这句说了它。引文里有、但引文本身没在块里定位到的,起点退回整块里的第一处
fn locate_time(chunk: &str, quote: Option<(&str, Option<(i32, i32)>)>, words: &str) -> Option<i32> {
let (q, span) = quote?;
- let (inner, _) = locate(q, words)?;
- match span {
- Some((start, _)) => Some(start + inner),
- None => locate(chunk, words).map(|(s, _)| s),
+ if let Some((inner, _)) = locate(q, words) {
+ return match span {
+ Some((start, _)) => Some(start + inner),
+ None => locate(chunk, words).map(|(s, _)| s),
+ };
+ }
+ // 表格的一行:期数写在表头行里,不在这一行里;这一块就是这张表(分块器让表头
+ // 跟着每一块走),所以在整块里找。判据是结构的:引文是一行 `|` 开头的表格行
+ if q.trim_start().starts_with('|') {
+ return locate(chunk, words).map(|(s, _)| s);
}
+ None
}
/// 名字的查找键:空白折叠、小写。陈述里写的名字和 `e` 里列的名字要一字不差,
@@ -368,6 +375,31 @@ pub(crate) async fn run_open(
},
None => None,
};
+ // 短语就是值、短语就是主语:表格丢了列头(或没有说明句)时模型的两种写法
+ // (#743 的财报长文里各占一类)。陈述照落——值和主语都在,信息没丢——只记一笔
+ // 让它可量;只比字面,不认词
+ let same = |a: &str, b: &str| a.trim().eq_ignore_ascii_case(b.trim());
+ if value.is_some_and(|v| same(v, phrase)) {
+ drop_signal(
+ state,
+ kb_id,
+ document_id,
+ reason::PHRASE_IS_VALUE,
+ "kept, but the phrase is the value itself",
+ Some(phrase),
+ )
+ .await;
+ } else if same(&s.subject, phrase) {
+ drop_signal(
+ state,
+ kb_id,
+ document_id,
+ reason::PHRASE_IS_SUBJECT,
+ "kept, but the phrase is the subject's own name",
+ Some(phrase),
+ )
+ .await;
+ }
let value_json;
let fact_object = match (object, value) {
(Some(o), _) if o == subject => {
@@ -734,6 +766,19 @@ mod tests {
assert_eq!(locate_time(text, None, "2019"), None);
}
+ #[test]
+ fn a_table_row_takes_its_period_from_the_header_in_the_same_chunk() {
+ let text = "| | Q2 FY27 | Q1 FY27 |\n| --- | --- | --- |\n| Revenue | $96,221 | $81,615 |";
+ let row = ("| Revenue | $96,221 | $81,615 |", Some((46, 76)));
+ // 期数在表头行里,不在这一行里:表格行在整块里找
+ assert_eq!(locate_time(text, Some(row), "Q2 FY27"), Some(5));
+ // 不是表格行的引文还是只认自己那句
+ assert_eq!(
+ locate_time(text, Some(("Revenue was up.", None)), "Q2 FY27"),
+ None
+ );
+ }
+
#[test]
fn names_match_up_to_whitespace_and_case() {
assert_eq!(name_key(" Harbor Bridge "), "harbor bridge");
diff --git a/crates/utopia-store/src/extraction_drops.rs b/crates/utopia-store/src/extraction_drops.rs
index 15cf8021..12c575cf 100644
--- a/crates/utopia-store/src/extraction_drops.rs
+++ b/crates/utopia-store/src/extraction_drops.rs
@@ -39,6 +39,13 @@ pub mod reason {
pub const TIME_NOT_IN_QUOTE: &str = "time_not_in_quote";
/// 陈述、属性或名字指着一个回复里不存在的实体编号或已知句柄:跳过这一条
pub const UNKNOWN_REF: &str = "unknown_ref";
+ /// 短语就是那个值(「director nominees —ten (10)→ ten (10)」):表格丢了列头时模型
+ /// 的写法之一。陈述照落(主语和值都在,信息没丢),记一笔让它可量。
+ /// 纯结构判断:两串字去掉首尾空白后相同
+ pub const PHRASE_IS_VALUE: &str = "phrase_is_value";
+ /// 短语就是主语的名字(「实物商品网上零售额 —实物商品网上零售额→ 127878亿元」):
+ /// 同上,另一种写法,同样照落
+ pub const PHRASE_IS_SUBJECT: &str = "phrase_is_subject";
}
pub async fn record(
diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts
index a2a48450..673ca27f 100644
--- a/web/src/i18n/en.ts
+++ b/web/src/i18n/en.ts
@@ -509,6 +509,8 @@ export const en = {
quote_not_in_chunk: "Kept, but the quoted sentence is not in the text verbatim",
time_not_in_quote: "A time mention's words are not in the text",
unknown_ref: "The item points at nothing in the reply",
+ phrase_is_value: "Kept, but the phrase is the value itself",
+ phrase_is_subject: "Kept, but the phrase is the subject's own name",
} as Record<string, string>,
// 来源级重抽:不危险,只是费时费钱——轻确认,文案直说成本与保留项
reExtractSource: "Re-extract",
diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts
index ec8fe671..ca6cb820 100644
--- a/web/src/i18n/zh.ts
+++ b/web/src/i18n/zh.ts
@@ -460,6 +460,8 @@ export const zh: Strings = {
quote_not_in_chunk: "已保留,但引文不是原文原样",
time_not_in_quote: "时间词不在原文里",
unknown_ref: "这一条指向的东西不在回复里",
+ phrase_is_value: "已保留,但短语就是那个值",
+ phrase_is_subject: "已保留,但短语就是主语的名字",
},
reExtractSource: "重新抽取",
reExtractTitle: "重新抽取这个来源?",