World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
A table keeps its headings, its sections and its caption (#744)
WaylandYangfc456184bb6b
diff --git a/Cargo.lock b/Cargo.lockindex 66de830c..e09af1e8 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -5425,6 +5425,7 @@ dependencies = ["calamine","chardetng","csv",+ "dom_query","dom_smoothie","encoding_rs","flate2",diff --git a/Cargo.toml b/Cargo.tomlindex bb197b5d..950b5787 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -78,6 +78,7 @@ cron = "0.12"infer = "0.22"sha2 = "0.11"dom_smoothie = "0.16"+dom_query = "0.26"htmd = "0.5"pulldown-cmark = { version = "0.13", default-features = false }chardetng = "1"diff --git a/crates/utopia-extract/src/open.rs b/crates/utopia-extract/src/open.rsindex 895c0f12..f2b140cf 100644--- a/crates/utopia-extract/src/open.rs+++ b/crates/utopia-extract/src/open.rs@@ -145,8 +145,14 @@ what is measured or stated (\"area\", \"was completed\", \"joined\", \"占地面literal as written. Wording that names or describes something is not a value: that something \goes in \"e\" and is linked by a statement with an object. \"1,200 beekeepers joined\" is the \thing \"beekeepers\" with \"joined\" = \"1,200\". Write every figure the passage states. In a \-table, a cell is a statement about its row's thing whose phrase is the column heading; leave \-out cells whose column heading is not in the passage.\n\+table, a cell is a statement about its row's thing whose phrase is the column heading. When \+the column heading names one time (a date, a quarter, a period), it is when instead, the \+statement is about the thing the table's caption names (the company of a financial statement) \+and the phrase is the row label as written, path included (\"Operating expenses › Research and \+development\"); a heading that names a comparison between periods (a change from the quarter \+before, a change from a year ago) is not a time, it stays the phrase; a unit the caption or a \+heading gives (\"In millions\") is a qualifier on each such statement. Leave out cells whose \+column heading is not in the passage.\n\A list is one statement per member, and so is a subject or object that joins several things \(\"the city and the county funded the bridge\" is two statements).\n\3. Nothing in a sentence is dropped. When more than two things take part, or the link carries \diff --git a/crates/utopia-ingest/Cargo.toml b/crates/utopia-ingest/Cargo.tomlindex 8bb639a0..12e97412 100644--- a/crates/utopia-ingest/Cargo.toml+++ b/crates/utopia-ingest/Cargo.toml@@ -13,6 +13,7 @@ pdf-extract.workspace = truecalamine.workspace = truehtmd.workspace = truedom_smoothie.workspace = true+dom_query.workspace = truethiserror.workspace = truepercent-encoding = "2.3.2"chardetng.workspace = truediff --git a/crates/utopia-ingest/src/chunker.rs b/crates/utopia-ingest/src/chunker.rsindex e6a58795..031e7c15 100644--- a/crates/utopia-ingest/src/chunker.rs+++ b/crates/utopia-ingest/src/chunker.rs@@ -47,6 +47,8 @@ pub const BUDGET_TOKENS: usize = 300;/// 说明句最长多少字节还算说明句:表上面那一段要短、或者以冒号结尾,/// 才当成表的一部分带着走;一整段分析不是说明句const CAPTION_MAX_BYTES: usize = 200;+/// 表前最多几段短说明一起当说明句+const CAPTION_RUN_MAX: usize = 5;/// 不到这么多 token 的一段(页码、脚注标记)不单独成块,并进上一块const TINY_TOKENS: usize = 8;@@ -100,11 +102,24 @@ fn units(text: &str, blocks: Vec<Block>) -> Vec<Unit> {// 提名人,结果如下:」后面是十张表,从前只有第一张带着它;黄仁勋// 那张在另一块里,四个票数都对,却没有一句话说这是在投什么,// 「当选董事」那条边就没了。一串到别的东西(段落、标题)出现为止。- let caption_like = b.range.len() <= CAPTION_MAX_BYTES- || text[b.range.clone()].trim_end().ends_with(':');- let mut j = i + 1;+ //+ // - 表前**连续几段**短说明一起算:财报的「NVIDIA CORPORATION」「CONDENSED+ // CONSOLIDATED STATEMENTS OF INCOME」「(In millions)」「(Unaudited)」是四个+ // 段落,只带最后一段的话,模型看到的表不知道是哪张报表、什么单位。+ // 一串最多五段,中间不隔别的东西+ let caption_like = |blk: &Block| {+ matches!(blk.kind, Kind::Paragraph)+ && (blk.range.len() <= CAPTION_MAX_BYTES+ || text[blk.range.clone()].trim_end().ends_with(':'))+ };+ let mut k = i + 1;+ while k - i < CAPTION_RUN_MAX && blocks.get(k).is_some_and(&caption_like) {+ k += 1;+ }+ let caption = b.range.start..blocks[k - 1].range.end;+ let mut j = k;let mut attached = 0usize;- if caption_like {+ if caption_like(b) {loop {while matches!(blocks.get(j),@@ -123,7 +138,7 @@ fn units(text: &str, blocks: Vec<Block>) -> Vec<Unit> {break;};out.push(Unit::Table {- caption: Some(b.range.clone()),+ caption: Some(caption.clone()),head: head.clone(),rows: rows.clone(),});@@ -132,6 +147,7 @@ fn units(text: &str, blocks: Vec<Block>) -> Vec<Unit> {}}if attached == 0 {+ // 不是说明句:这一段照常;后面那几段下一轮各自再看out.push(Unit::Text(b.range.clone()));} else {i = j - 1;@@ -769,6 +785,45 @@ The tenant pays rent monthly.}}+ /// 财报的形状:公司名、报表名、单位、「未经审计」四段短说明,然后是表。+ /// 四段都是说明句,切开的每一块都带着全部四段。+ #[test]+ fn a_run_of_short_paragraphs_before_a_table_is_its_caption() {+ let text = format!(+ "Prose that is long enough not to be a caption, going on about the quarter and the outlook and the products and the customers.\n\n\+ NVIDIA CORPORATION\n\nCONDENSED CONSOLIDATED STATEMENTS OF INCOME\n\n(In millions)\n\n(Unaudited)\n\n{}",+ table(30)+ );+ let pieces = chunk_with_budget(&text, 120);+ let with_table: Vec<&ChunkPiece> = pieces+ .iter()+ .filter(|p| p.text.contains("| --- |"))+ .collect();+ assert!(with_table.len() >= 2, "表该被切成几块: {}", pieces.len());+ for p in &with_table {+ assert!(+ p.text.contains("NVIDIA CORPORATION"),+ "每块都带公司名: {}",+ p.text+ );+ assert!(+ p.text.contains("STATEMENTS OF INCOME"),+ "每块都带报表名: {}",+ p.text+ );+ assert!(+ p.text.contains("(Unaudited)"),+ "每块都带最后一段: {}",+ p.text+ );+ assert!(+ !p.text.contains("Prose that is long"),+ "长段落不是说明句: {}",+ p.text+ );+ }+ }+/// 十位董事的形状:一句说明、一串表。每张表不论落在哪一块都看得见那句说明;/// 同一块里的几张表,说明只念一遍。#[test]diff --git a/crates/utopia-ingest/src/html.rs b/crates/utopia-ingest/src/html.rsindex 2a9895f0..a6c42d49 100644--- a/crates/utopia-ingest/src/html.rs+++ b/crates/utopia-ingest/src/html.rs@@ -488,7 +488,10 @@ fn prune_empty_table_columns(markdown: &str) -> String {}fn markdown_from_html(html: &str) -> Result<String, HtmlError> {- let html = &promote_first_row_headers(html);+ // 最外层的表先从 DOM 渲染成带真表头的 Markdown(见 `table`),htmd 只看到占位段落;+ // 套着的表还走老路,所以第一行提表头的补丁留着给它们+ let (html, tables) = crate::table::lift_tables(html);+ let html = &promote_first_row_headers(&html);let markdown = htmd::HtmlToMarkdown::builder().skip_tags(vec!["script", "style", "iframe", "object", "embed", "img", "svg", "math",@@ -505,6 +508,7 @@ fn markdown_from_html(html: &str) -> Result<String, HtmlError> {{return Err(HtmlError::Interstitial);}+ let markdown = crate::table::restore_tables(&markdown, &tables);normalize_markdown(&prune_empty_table_columns(&markdown))}diff --git a/crates/utopia-ingest/src/lib.rs b/crates/utopia-ingest/src/lib.rsindex 8e940fcd..94a76e44 100644--- a/crates/utopia-ingest/src/lib.rs+++ b/crates/utopia-ingest/src/lib.rs@@ -169,6 +169,7 @@ pub mod ontology_rdf;mod parsers;pub mod provenance;mod reading;+mod table;pub mod transcript;pub use chunker::{chunk_segments, chunk_text, chunk_with_budget, ChunkPiece, BUDGET_TOKENS};diff --git a/crates/utopia-ingest/src/table.rs b/crates/utopia-ingest/src/table.rsnew file mode 100644index 00000000..97f82fd7--- /dev/null+++ b/crates/utopia-ingest/src/table.rs@@ -0,0 +1,681 @@+//! 表格从 DOM 直接渲染成带真表头的 Markdown 表(0039/0040 那一层的活)。+//!+//! 为什么不交给 htmd:SEC/XBRL 报表的表一个 `<th>` 都没有。标题行(公司名、报表名、+//! 单位)是 colspan 跨全宽的普通行,列头("July 26," / "2026")拆在两行里,每个数字前后+//! 各有一格 "$" 或 ")",列与列之间是空的占位格。此前的补丁把第一行提成表头,于是被切开+//! 的每一块表头都是「| NVIDIA CORPORATION | | |」,真列头一块都没带;模型面对+//! 「Accounts receivable, net | 63,059 | 38,466」只能写出「短语 = 数字」或者不写。财报+//! 长文在 #743 下 56 条错说法里 38 条、64 条读不通里 43 条出自这里。+//!+//! 这里只看结构,不认任何词:跨列格、空列、只含符号的格、只有一格的行、行标签的左内+//! 边距、格子里有没有数字和字母。做四件事:标题行提成表前的说明句;多行列头按列拼成+//! 一行;只有标签没有值的行当成小节,折进后面每行的标签里(「Current assets ›+//! Accounts receivable, net」);"$"、")" 这类符号格并回相邻的数字格。渲染出的表交给+//! `blocks` / `chunker`,它们本来就让说明句和表头跟着每一块走。+//!+//! 只处理最外层且不含内层表的表;套着的表留给 htmd 原路。++use dom_query::{Document, Selection};++const PLACEHOLDER: &str = "UTOPIATABLE";+const PATH_SEP: &str = " › ";++/// 一格:文字、跨了几列、左内边距(pt,取整)、在网格里的起始列。+#[derive(Clone, Debug, Default)]+struct Cell {+ text: String,+ span: usize,+ pad: u32,+ col: usize,+}++#[derive(Clone, Debug)]+struct Row {+ cells: Vec<Cell>,+ /// 网格总宽(含跨列)+ width: usize,+}++impl Row {+ /// 覆盖第 k 列的那个非空格+ fn covering(&self, k: usize) -> Option<&Cell> {+ self.cells+ .iter()+ .find(|c| !c.text.is_empty() && c.col <= k && k < c.col + c.span)+ }+}++#[derive(Debug, PartialEq, Eq)]+enum Kind {+ Skip,+ Caption,+ Header,+ Section,+ Data,+}++/// 把最外层且不含内层表的表换成占位段落;返回改过的 HTML 和每个占位对应的 Markdown。+pub fn lift_tables(html: &str) -> (String, Vec<String>) {+ let doc = Document::from(html);+ let mut rendered: Vec<String> = Vec::new();+ let mut targets: Vec<Selection<'_>> = Vec::new();+ // 上一张表的列头:分页符把一张报表劈成两个 <table> 时,后半张没有表头,+ // 列数又一样,就是同一张表的续表,接着用前一张的列头+ let mut last_headers: Vec<String> = Vec::new();+ for tbl in doc.select("table").iter() {+ let nested_in = tbl.nodes().first().is_some_and(|n| {+ n.ancestors(None)+ .iter()+ .any(|a| a.node_name().as_deref() == Some("table"))+ });+ if nested_in || tbl.select("table").exists() {+ continue;+ }+ if let Some((md, headers)) = render_table(&tbl, &last_headers) {+ rendered.push(md);+ targets.push(tbl);+ last_headers = headers;+ }+ }+ if rendered.is_empty() {+ return (html.to_string(), rendered);+ }+ for (i, tbl) in targets.iter().enumerate() {+ tbl.replace_with_html(format!("<p>{PLACEHOLDER}{i}</p>"));+ }+ (doc.html().to_string(), rendered)+}++/// 把 htmd 转出的 Markdown 里的占位行换回渲染好的表。+pub fn restore_tables(markdown: &str, tables: &[String]) -> String {+ if tables.is_empty() {+ return markdown.to_string();+ }+ let extra: usize = tables.iter().map(String::len).sum();+ let mut out = String::with_capacity(markdown.len() + extra);+ for line in markdown.lines() {+ let hit = line+ .trim()+ .strip_prefix(PLACEHOLDER)+ .and_then(|rest| rest.parse::<usize>().ok())+ .and_then(|i| tables.get(i));+ match hit {+ Some(md) => {+ out.push('\n');+ out.push_str(md);+ out.push('\n');+ }+ None => out.push_str(line),+ }+ out.push('\n');+ }+ out+}++fn hidden(style: &str) -> bool {+ let s: String = style.chars().filter(|c| !c.is_whitespace()).collect();+ s.to_ascii_lowercase().contains("display:none")+}++/// 左内边距:`padding-left: 12pt`、`text-indent: 6pt`,或 `padding: a b c d` 的第四个值。+fn padding_left(style: &str) -> u32 {+ let lower = style.to_ascii_lowercase();+ let number_at = |s: &str| -> Option<f32> {+ let s = s.trim_start();+ let end = s+ .find(|c: char| !(c.is_ascii_digit() || c == '.'))+ .unwrap_or(s.len());+ s[..end].parse::<f32>().ok()+ };+ for key in ["padding-left:", "text-indent:"] {+ if let Some(i) = lower.find(key) {+ if let Some(v) = number_at(&lower[i + key.len()..]) {+ return v.round() as u32;+ }+ }+ }+ if let Some(i) = lower.find("padding:") {+ let rest = lower[i + "padding:".len()..]+ .split(';')+ .next()+ .unwrap_or("");+ let parts: Vec<&str> = rest.split_whitespace().collect();+ let left = match parts.len() {+ 4 => parts.get(3),+ 2 | 3 => parts.get(1),+ 1 => parts.first(),+ _ => None,+ };+ if let Some(v) = left.and_then(|p| number_at(p)) {+ return v.round() as u32;+ }+ }+ 0+}++fn clean(text: &str) -> String {+ text.replace('\u{a0}', " ")+ .split_whitespace()+ .collect::<Vec<_>>()+ .join(" ")+}++fn attr_usize(cell: &Selection<'_>, name: &str) -> usize {+ cell.attr(name)+ .and_then(|v| v.trim().parse::<usize>().ok())+ .unwrap_or(1)+ .max(1)+}++fn is_numeric(text: &str) -> bool {+ text.chars().any(|c| c.is_ascii_digit()) && !text.chars().any(char::is_alphabetic)+}++/// 该并回邻格的符号格:货币符号和左括号贴到后一格前面,右括号和百分号贴到前一格后面。+/// 破折号那类「空值」占位符不算——它自己就是这一格的内容,并过去会把整行的值挪位+fn merge_direction(text: &str) -> Option<bool> {+ let mut chars = text.chars();+ let first = chars.next()?;+ if chars.next().is_some() {+ return None;+ }+ if first == '(' || is_currency(first) {+ Some(true)+ } else if first == ')' || first == '%' {+ Some(false)+ } else {+ None+ }+}++fn is_currency(c: char) -> bool {+ matches!(c, '$' | '€' | '£' | '¥' | '₩' | '₹' | '¢' | '¥')+}++/// 读出网格:按 `tr` 逐行,跨行的格顺着往下带。+fn grid(tbl: &Selection<'_>) -> Vec<Row> {+ let mut rows: Vec<Row> = Vec::new();+ // (列, 格, 还剩几行)+ let mut carry: Vec<(usize, Cell, usize)> = Vec::new();+ for tr in tbl.select("tr").iter() {+ let mut cells: Vec<Cell> = Vec::new();+ let mut col = 0usize;+ for cell in tr.select("td, th").iter() {+ let style = cell+ .attr("style")+ .map(|s| s.to_string())+ .unwrap_or_default();+ if hidden(&style) {+ continue;+ }+ place_carried(&mut col, &mut cells, &mut carry);+ let span = attr_usize(&cell, "colspan");+ let rowspan = attr_usize(&cell, "rowspan");+ let c = Cell {+ text: clean(&cell.text()),+ span,+ pad: padding_left(&style),+ col,+ };+ if rowspan > 1 {+ carry.push((col, c.clone(), rowspan - 1));+ }+ col += span;+ cells.push(c);+ }+ place_carried(&mut col, &mut cells, &mut carry);+ rows.push(Row { cells, width: col });+ }+ rows+}++fn place_carried(col: &mut usize, cells: &mut Vec<Cell>, carry: &mut Vec<(usize, Cell, usize)>) {+ while let Some(pos) = carry.iter().position(|(c, _, _)| *c == *col) {+ let (_, mut cell, left) = carry[pos].clone();+ cell.col = *col;+ *col += cell.span;+ cells.push(cell);+ if left <= 1 {+ carry.remove(pos);+ } else {+ carry[pos].2 = left - 1;+ }+ }+}++/// 一行是什么:只看非空格的个数、位置、跨度,和格子里有没有数字。+fn kind(row: &Row, headers_seen: bool, data_seen: bool, table_width: usize) -> Kind {+ let nonempty: Vec<&Cell> = row.cells.iter().filter(|c| !c.text.is_empty()).collect();+ let Some(first) = nonempty.first() else {+ return Kind::Skip;+ };+ if nonempty.len() == 1 {+ let wide = first.span >= 2 && first.span * 2 >= table_width.max(2);+ if is_numeric(&first.text) {+ return if data_seen { Kind::Data } else { Kind::Header };+ }+ if !headers_seen && !data_seen {+ // 表头和数据都还没来:靠左或跨宽的一格文字是标题,不靠左的是一根列头+ return if wide || first.col == 0 {+ Kind::Caption+ } else {+ Kind::Header+ };+ }+ return Kind::Section;+ }+ let label = !is_numeric(&first.text);+ let values_after = nonempty[1..].iter().any(|c| is_numeric(&c.text));+ // 有标签跟着数字的是数据行;最左一格就有内容的整排数字也是(年份列的表)。+ // 最左几格空着、全是数字的一排("2026 | 2026")是列头的第二行+ if values_after && (label || first.col == 0 || data_seen) {+ return Kind::Data;+ }+ if !data_seen {+ return Kind::Header;+ }+ if label {+ // 数据行之后又一排没有数字的格:当成一条小节+ return Kind::Section;+ }+ Kind::Data+}++/// 把只含符号的格并回邻格:"$" 和 "(" 贴到后一个非空格前面,")" 和 "%" 贴到前一个后面。+fn merge_symbols(row: &mut Row) {+ let n = row.cells.len();+ for i in 0..n {+ let Some(opening) = merge_direction(&row.cells[i].text) else {+ continue;+ };+ let sym = row.cells[i].text.clone();+ let target = if opening {+ (i + 1..n).find(|&j| !row.cells[j].text.is_empty())+ } else {+ (0..i).rev().find(|&j| !row.cells[j].text.is_empty())+ };+ let Some(j) = target else { continue };+ if opening {+ row.cells[j].text = format!("{sym}{}", row.cells[j].text);+ } else {+ row.cells[j].text = format!("{}{sym}", row.cells[j].text);+ }+ row.cells[i].text.clear();+ }+}++fn escape(text: &str) -> String {+ text.replace('|', "\\|")+}++/// 缩进深度:先看标签格前面有几根空列(Workiva 报表用空格子缩进),再看左内边距+/// (另一些用 padding-left)。两者合成一个数,列在前、内边距在后+fn depth_of(cell: &Cell) -> u32 {+ cell.col as u32 * 1000 + cell.pad+}++/// 一张表 → 说明句(可选)+ 带真表头的 Markdown 表,连同它用的列头(给续表接着用)。+/// 表里什么都没有时返回 None。+fn render_table(tbl: &Selection<'_>, inherited: &[String]) -> Option<(String, Vec<String>)> {+ let mut rows = grid(tbl);+ let width = rows.iter().map(|r| r.width).max().unwrap_or(0);+ if width == 0 {+ return None;+ }+ let mut captions: Vec<String> = Vec::new();+ let mut headers: Vec<Row> = Vec::new();+ // (小节深度, 小节名, 已经见过比它深的行)。+ // 一个小节被同深度的数据行结束,只在它已经有过更深的子行之后:资产负债表里+ // 「Current assets:」的子行缩进了,同深度的「Property and equipment」是下一项;+ // 现金流量表里「Cash flows from operating activities:」的子行没缩进,同深度的+ // 「Net income」就是它的第一行+ let mut sections: Vec<(u32, String, bool)> = Vec::new();+ // (标签路径, 该行)+ let mut data: Vec<(String, Row)> = Vec::new();+ let (mut headers_seen, mut data_seen) = (false, false);+ for row in rows.iter_mut() {+ match kind(row, headers_seen, data_seen, width) {+ Kind::Skip => {}+ Kind::Caption => {+ if let Some(c) = row.cells.iter().find(|c| !c.text.is_empty()) {+ captions.push(c.text.clone());+ }+ }+ Kind::Header => {+ merge_symbols(row);+ headers.push(row.clone());+ headers_seen = true;+ }+ Kind::Section => {+ let Some(cell) = row.cells.iter().find(|c| !c.text.is_empty()) else {+ continue;+ };+ let depth = depth_of(cell);+ while sections.last().is_some_and(|(d, _, _)| *d >= depth) {+ sections.pop();+ }+ sections.push((+ depth,+ cell.text.trim_end_matches([':', ':']).to_string(),+ false,+ ));+ }+ Kind::Data => {+ merge_symbols(row);+ let label_cell = row.cells.iter().find(|c| !c.text.is_empty());+ let depth = label_cell.map(depth_of).unwrap_or(0);+ while sections+ .last()+ .is_some_and(|(d, _, deeper)| *d > depth || (*d == depth && *deeper))+ {+ sections.pop();+ }+ if let Some(top) = sections.last_mut() {+ if top.0 < depth {+ top.2 = true;+ }+ }+ let path: Vec<&str> = sections.iter().map(|(_, s, _)| s.as_str()).collect();+ data.push((path.join(PATH_SEP), row.clone()));+ data_seen = true;+ }+ }+ }+ if data.is_empty() && headers.is_empty() {+ return None;+ }++ // 标签列 = 第一根出现过数字的列左边的所有列;一个数字都没有的表,标签列只有最左那格。+ // 值列 = 标签列右边、数据行里出现过内容的列(表头有字但整列没数据的列不算)+ let label_end: usize = (0..width)+ .find(|&k| {+ data.iter()+ .any(|(_, r)| r.covering(k).is_some_and(|c| is_numeric(&c.text)))+ })+ .or_else(|| {+ data.iter()+ .filter_map(|(_, r)| {+ r.cells+ .iter()+ .find(|c| !c.text.is_empty())+ .map(|c| c.col + c.span)+ })+ .min()+ })+ .or_else(|| {+ headers+ .iter()+ .filter_map(|h| h.cells.iter().find(|c| !c.text.is_empty()).map(|c| c.col))+ .min()+ })+ .unwrap_or(0);+ let has_content = |k: usize| -> bool {+ if data.is_empty() {+ headers.iter().any(|h| h.covering(k).is_some())+ } else {+ data.iter().any(|(_, r)| r.covering(k).is_some())+ }+ };+ let value_cols: Vec<usize> = (label_end..width).filter(|&k| has_content(k)).collect();+ let value_cols: Vec<Vec<usize>> = collapse_spans(value_cols, &headers, &data);+ // 一根值列是一组网格列:取这一行落在这组里的第一个非空格+ let at = |r: &Row, group: &[usize]| -> String {+ group+ .iter()+ .find_map(|&k| r.covering(k))+ .map(|c| c.text.clone())+ .unwrap_or_default()+ };+ // 一行的标签:标签列里的格子按出现顺序拼起来+ let own_label = |r: &Row| -> String {+ let mut parts: Vec<&str> = Vec::new();+ for c in r+ .cells+ .iter()+ .filter(|c| !c.text.is_empty() && c.col < label_end)+ {+ if parts.last() != Some(&c.text.as_str()) {+ parts.push(c.text.as_str());+ }+ }+ parts.join(" ")+ };+ let column_header = |group: &[usize]| -> String {+ let mut parts: Vec<String> = Vec::new();+ for h in &headers {+ let t = at(h, group);+ if !t.is_empty() && parts.last() != Some(&t) {+ parts.push(t);+ }+ }+ parts.join(" ")+ };+ let mut header_texts: Vec<String> = value_cols.iter().map(|g| column_header(g)).collect();+ if headers.is_empty() && !data.is_empty() && inherited.len() == header_texts.len() {+ header_texts = inherited.to_vec();+ }+ // 标签列的表头:只算整格都落在标签列里的表头格,跨进值列的是值列的头+ let label_header: String = {+ let mut parts: Vec<String> = Vec::new();+ for h in &headers {+ for c in h+ .cells+ .iter()+ .filter(|c| !c.text.is_empty() && c.col + c.span <= label_end)+ {+ if parts.last() != Some(&c.text) {+ parts.push(c.text.clone());+ }+ }+ }+ parts.join(" ")+ };++ let mut lines: Vec<String> = Vec::new();+ if !captions.is_empty() {+ let mut caption = captions.join(" · ");+ if !caption.ends_with(':') && !caption.ends_with(':') {+ caption.push(':');+ }+ lines.push(caption);+ lines.push(String::new());+ }+ // 最左一列本身就是数字(年份列的表):没有标签列,行就是它的值+ let has_label_col = label_end > 0;+ let ncols = usize::from(has_label_col) + value_cols.len().max(1);+ let header_cells: Vec<String> = has_label_col+ .then(|| escape(&label_header))+ .into_iter()+ .chain(header_texts.iter().map(|h| escape(h)))+ .chain(std::iter::repeat(String::new()))+ .take(ncols)+ .collect();+ lines.push(format!("| {} |", header_cells.join(" | ")));+ lines.push(format!("|{}", " --- |".repeat(ncols)));+ for (sections, r) in &data {+ let mut cells: Vec<String> = Vec::new();+ if has_label_col {+ let own = own_label(r);+ let label = match (sections.is_empty(), own.is_empty()) {+ (true, _) => own,+ (false, true) => sections.clone(),+ (false, false) => format!("{sections}{PATH_SEP}{own}"),+ };+ cells.push(escape(&label));+ }+ cells.extend(value_cols.iter().map(|g| escape(&at(r, g))));+ while cells.len() < ncols {+ cells.push(String::new());+ }+ lines.push(format!("| {} |", cells.join(" | ")));+ }+ Some((lines.join("\n"), header_texts))+}++/// 跨列的格会让同一个值落在好几根网格列上,"$" 又常常单独占一格:相邻两列在每一行+/// 上要么由同一格覆盖、要么至少一边是空的,就是同一根值列,并成一组。+fn collapse_spans(cols: Vec<usize>, headers: &[Row], data: &[(String, Row)]) -> Vec<Vec<usize>> {+ let mut out: Vec<Vec<usize>> = Vec::new();+ for k in cols {+ let same_as_prev = out.last().and_then(|g| g.last().copied()).is_some_and(|p| {+ headers.iter().chain(data.iter().map(|(_, r)| r)).all(|r| {+ match (r.covering(p), r.covering(k)) {+ (Some(a), Some(b)) => a.col == b.col,+ _ => true,+ }+ })+ });+ match out.last_mut() {+ Some(g) if same_as_prev => g.push(k),+ _ => out.push(vec![k]),+ }+ }+ out+}++#[cfg(test)]+mod tests {+ use super::*;++ fn render(html: &str) -> String {+ let (_, tables) = lift_tables(html);+ tables.join("\n=====\n")+ }++ #[test]+ fn a_filing_table_gets_its_real_headers_and_its_section_path() {+ let html = r#"<table>+ <tr><td style="width:0.1%"></td><td style="width:40%"></td><td style="width:0.1%"></td><td style="width:1%"></td><td style="width:20%"></td><td style="width:1%"></td><td style="width:1%"></td><td style="width:20%"></td><td style="width:0.1%"></td><td colspan="3" style="display:none"></td></tr>+ <tr><td colspan="9" style="text-align:center">NVIDIA CORPORATION</td><td colspan="3" style="display:none"></td></tr>+ <tr><td colspan="9" style="text-align:center">CONDENSED CONSOLIDATED BALANCE SHEETS</td></tr>+ <tr><td colspan="9" style="text-align:center">(In millions)</td></tr>+ <tr><td colspan="3"></td><td colspan="3" style="text-align:center">July 26,</td><td colspan="3" style="text-align:center">January 25,</td></tr>+ <tr><td colspan="3"></td><td colspan="3" style="text-align:center">2026</td><td colspan="3" style="text-align:center">2026</td></tr>+ <tr><td colspan="3" style="padding:2pt 1pt 2pt 6pt">Current assets:</td><td colspan="3"></td><td colspan="3"></td></tr>+ <tr><td colspan="3" style="padding:2pt 1pt 2pt 12pt">Accounts receivable, net</td><td>$</td><td>63,059</td><td></td><td>$</td><td>38,466</td><td></td></tr>+ <tr><td colspan="3" style="padding:2pt 1pt 2pt 12pt">Inventories</td><td></td><td>31,575</td><td></td><td></td><td>21,403</td><td></td></tr>+ <tr><td colspan="3" style="padding:2pt 1pt 2pt 6pt">Total current assets</td><td></td><td>197,412</td><td></td><td></td><td>125,605</td><td></td></tr>+ </table>"#;+ let md = render(html);+ assert!(+ md.starts_with(+ "NVIDIA CORPORATION · CONDENSED CONSOLIDATED BALANCE SHEETS · (In millions):\n"+ ),+ "标题行提成表前的说明句: {md}"+ );+ assert!(+ md.contains("| | July 26, 2026 | January 25, 2026 |"),+ "两行列头拼成一行: {md}"+ );+ assert!(+ md.contains("| Current assets › Accounts receivable, net | $63,059 | $38,466 |"),+ "小节折进标签、美元符并回数字: {md}"+ );+ assert!(+ md.contains("| Current assets › Inventories | 31,575 | 21,403 |"),+ "{md}"+ );+ assert!(+ md.contains("| Total current assets | 197,412 | 125,605 |"),+ "合计行回到小节外: {md}"+ );+ assert!(!md.contains("Current assets:"), "小节行自己不再出现: {md}");+ }++ #[test]+ fn a_lone_label_before_everything_is_the_caption_and_rows_keep_their_values() {+ let html = "<table><tr><td colspan=\"2\">a. Tench Coxe</td></tr>\+ <tr><td>Number of shares For</td><td>15,411,252,412</td></tr>\+ <tr><td>Number of shares Against</td><td>1,399,727</td></tr></table>";+ let md = render(html);+ assert!(md.starts_with("a. Tench Coxe:\n"), "{md}");+ assert!(+ md.contains("| Number of shares For | 15,411,252,412 |"),+ "{md}"+ );+ assert!(+ md.contains("| Number of shares Against | 1,399,727 |"),+ "{md}"+ );+ }++ #[test]+ fn a_first_row_of_words_is_the_header_when_the_table_has_none() {+ let html = "<table><tr><td>Revenue</td><td>Q2 FY27</td><td>Q1 FY27</td></tr>\+ <tr><td>total</td><td>$96,221</td><td>$81,615</td></tr></table>";+ let md = render(html);+ assert!(md.contains("| Revenue | Q2 FY27 | Q1 FY27 |"), "{md}");+ assert!(md.contains("| total | $96,221 | $81,615 |"), "{md}");+ }++ #[test]+ fn hidden_cells_and_empty_columns_do_not_survive() {+ let html = "<table><tr><td>Margin</td><td style=\"display:none\">ghost</td><td></td><td>75.0</td><td></td></tr></table>";+ let md = render(html);+ assert!(!md.contains("ghost"), "{md}");+ assert!(md.contains("| Margin | 75.0 |"), "{md}");+ }++ #[test]+ fn a_closing_paren_in_its_own_cell_rejoins_the_number() {+ let html =+ "<table><tr><td>Other</td><td>(5,497</td><td>)</td><td>387</td><td></td></tr></table>";+ let md = render(html);+ assert!(md.contains("| Other | (5,497) | 387 |"), "{md}");+ }++ #[test]+ fn a_dash_that_means_nothing_stays_in_its_own_cell() {+ let html =+ "<table><tr><td>Other</td><td>112</td><td>—</td><td>31</td><td>—</td></tr></table>";+ let md = render(html);+ assert!(md.contains("| Other | 112 | — | 31 | — |"), "{md}");+ }++ #[test]+ fn a_headerless_table_after_a_page_break_takes_the_headers_before_it() {+ let html = "<table>\+ <tr><td></td><td>Q2 FY27</td><td>Q1 FY27</td></tr>\+ <tr><td>Revenue</td><td>96,221</td><td>81,615</td></tr></table>\+ <hr>\+ <table><tr><td colspan=\"3\">Cash flows from financing activities:</td></tr>\+ <tr><td>Dividends paid</td><td>(6,047)</td><td>(244)</td></tr></table>";+ let (_, tables) = lift_tables(html);+ assert_eq!(tables.len(), 2, "{tables:?}");+ assert!(+ tables[1].starts_with("Cash flows from financing activities:\n"),+ "{}",+ tables[1]+ );+ assert!(+ tables[1].contains("| | Q2 FY27 | Q1 FY27 |"),+ "续表接着用前一张的列头: {}",+ tables[1]+ );+ assert!(+ tables[1].contains("| Dividends paid | (6,047) | (244) |"),+ "{}",+ tables[1]+ );+ }++ #[test]+ fn nested_tables_are_left_to_the_markdown_converter() {+ let html =+ "<table><tr><td><table><tr><td>inner</td><td>1</td></tr></table></td></tr></table>";+ let (out, tables) = lift_tables(html);+ assert!(tables.is_empty(), "{tables:?}");+ assert_eq!(out, html);+ }++ #[test]+ fn placeholders_come_back_as_tables() {+ let md = "before\n\nUTOPIATABLE0\n\nafter";+ let table = "cap:\n\n| | a |\n| --- | --- |\n| x | 1 |".to_string();+ let out = restore_tables(md, &[table]);+ assert!(+ out.contains("before\n\n\ncap:\n\n| | a |\n| --- | --- |\n| x | 1 |\n\n\nafter"),+ "{out}"+ );+ }+}diff --git a/crates/utopia-server/src/extraction_open.rs b/crates/utopia-server/src/extraction_open.rsindex a5b94f8c..59dabb46 100644--- a/crates/utopia-server/src/extraction_open.rs+++ b/crates/utopia-server/src/extraction_open.rs@@ -61,11 +61,18 @@ fn locate(hay: &str, needle: &str) -> Option<(i32, i32)> {/// 整块里搜得到不等于这句说了它。引文里有、但引文本身没在块里定位到的,起点退回整块里的第一处fn locate_time(chunk: &str, quote: Option<(&str, Option<(i32, i32)>)>, words: &str) -> Option<i32> {let (q, span) = quote?;- let (inner, _) = locate(q, words)?;- match span {- Some((start, _)) => Some(start + inner),- None => locate(chunk, words).map(|(s, _)| s),+ if let Some((inner, _)) = locate(q, words) {+ return match span {+ Some((start, _)) => Some(start + inner),+ None => locate(chunk, words).map(|(s, _)| s),+ };+ }+ // 表格的一行:期数写在表头行里,不在这一行里;这一块就是这张表(分块器让表头+ // 跟着每一块走),所以在整块里找。判据是结构的:引文是一行 `|` 开头的表格行+ if q.trim_start().starts_with('|') {+ return locate(chunk, words).map(|(s, _)| s);}+ None}/// 名字的查找键:空白折叠、小写。陈述里写的名字和 `e` 里列的名字要一字不差,@@ -368,6 +375,31 @@ pub(crate) async fn run_open(},None => None,};+ // 短语就是值、短语就是主语:表格丢了列头(或没有说明句)时模型的两种写法+ // (#743 的财报长文里各占一类)。陈述照落——值和主语都在,信息没丢——只记一笔+ // 让它可量;只比字面,不认词+ let same = |a: &str, b: &str| a.trim().eq_ignore_ascii_case(b.trim());+ if value.is_some_and(|v| same(v, phrase)) {+ drop_signal(+ state,+ kb_id,+ document_id,+ reason::PHRASE_IS_VALUE,+ "kept, but the phrase is the value itself",+ Some(phrase),+ )+ .await;+ } else if same(&s.subject, phrase) {+ drop_signal(+ state,+ kb_id,+ document_id,+ reason::PHRASE_IS_SUBJECT,+ "kept, but the phrase is the subject's own name",+ Some(phrase),+ )+ .await;+ }let value_json;let fact_object = match (object, value) {(Some(o), _) if o == subject => {@@ -734,6 +766,19 @@ mod tests {assert_eq!(locate_time(text, None, "2019"), None);}+ #[test]+ fn a_table_row_takes_its_period_from_the_header_in_the_same_chunk() {+ let text = "| | Q2 FY27 | Q1 FY27 |\n| --- | --- | --- |\n| Revenue | $96,221 | $81,615 |";+ let row = ("| Revenue | $96,221 | $81,615 |", Some((46, 76)));+ // 期数在表头行里,不在这一行里:表格行在整块里找+ assert_eq!(locate_time(text, Some(row), "Q2 FY27"), Some(5));+ // 不是表格行的引文还是只认自己那句+ assert_eq!(+ locate_time(text, Some(("Revenue was up.", None)), "Q2 FY27"),+ None+ );+ }+#[test]fn names_match_up_to_whitespace_and_case() {assert_eq!(name_key(" Harbor Bridge "), "harbor bridge");diff --git a/crates/utopia-store/src/extraction_drops.rs b/crates/utopia-store/src/extraction_drops.rsindex 15cf8021..12c575cf 100644--- a/crates/utopia-store/src/extraction_drops.rs+++ b/crates/utopia-store/src/extraction_drops.rs@@ -39,6 +39,13 @@ pub mod reason {pub const TIME_NOT_IN_QUOTE: &str = "time_not_in_quote";/// 陈述、属性或名字指着一个回复里不存在的实体编号或已知句柄:跳过这一条pub const UNKNOWN_REF: &str = "unknown_ref";+ /// 短语就是那个值(「director nominees —ten (10)→ ten (10)」):表格丢了列头时模型+ /// 的写法之一。陈述照落(主语和值都在,信息没丢),记一笔让它可量。+ /// 纯结构判断:两串字去掉首尾空白后相同+ pub const PHRASE_IS_VALUE: &str = "phrase_is_value";+ /// 短语就是主语的名字(「实物商品网上零售额 —实物商品网上零售额→ 127878亿元」):+ /// 同上,另一种写法,同样照落+ pub const PHRASE_IS_SUBJECT: &str = "phrase_is_subject";}pub async fn record(diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.tsindex a2a48450..673ca27f 100644--- a/web/src/i18n/en.ts+++ b/web/src/i18n/en.ts@@ -509,6 +509,8 @@ export const en = {quote_not_in_chunk: "Kept, but the quoted sentence is not in the text verbatim",time_not_in_quote: "A time mention's words are not in the text",unknown_ref: "The item points at nothing in the reply",+ phrase_is_value: "Kept, but the phrase is the value itself",+ phrase_is_subject: "Kept, but the phrase is the subject's own name",} as Record<string, string>,// 来源级重抽:不危险,只是费时费钱——轻确认,文案直说成本与保留项reExtractSource: "Re-extract",diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.tsindex ec8fe671..ca6cb820 100644--- a/web/src/i18n/zh.ts+++ b/web/src/i18n/zh.ts@@ -460,6 +460,8 @@ export const zh: Strings = {quote_not_in_chunk: "已保留,但引文不是原文原样",time_not_in_quote: "时间词不在原文里",unknown_ref: "这一条指向的东西不在回复里",+ phrase_is_value: "已保留,但短语就是那个值",+ phrase_is_subject: "已保留,但短语就是主语的名字",},reExtractSource: "重新抽取",reExtractTitle: "重新抽取这个来源?",