World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Preserve Notion page bodies on read failures and import table cells (#1101)
Daniel654141aa1c57
diff --git a/crates/utopia-server/src/ingest_sources.rs b/crates/utopia-server/src/ingest_sources.rsindex b0cedc1d..62f81c69 100644--- a/crates/utopia-server/src/ingest_sources.rs+++ b/crates/utopia-server/src/ingest_sources.rs@@ -1165,7 +1165,7 @@ async fn sync_notion(state: &AppState, source: &Source) -> anyhow::Result<SyncSt.map(str::trim).filter(|s| !s.is_empty());- let (pages, truncated) = crate::notion::fetch(token, query).await?;+ let (pages, truncated, unread) = crate::notion::fetch(token, query).await?;if truncated {tracing::warn!("页面数到达单次上限,其余留给下一次同步");}@@ -1185,6 +1185,21 @@ async fn sync_notion(state: &AppState, source: &Source) -> anyhow::Result<SyncSt.await?;stats.absorb(action);}+ // 读得出来的页已经进库了;读不出来的那几页让这一轮记成失败并点名,+ // 不然它们停在旧版本上没有人知道+ if !unread.is_empty() {+ anyhow::bail!(+ "{} Notion page(s) synced; {} could not be read and keep their stored content: {}",+ stats.total(),+ unread.len(),+ unread+ .iter()+ .take(3)+ .map(String::as_str)+ .collect::<Vec<_>>()+ .join("; ")+ );+ }Ok(stats)}diff --git a/crates/utopia-server/src/notion.rs b/crates/utopia-server/src/notion.rsindex 344e3a54..e53f1390 100644--- a/crates/utopia-server/src/notion.rs+++ b/crates/utopia-server/src/notion.rs@@ -167,7 +167,12 @@ fn retry_after(header: Option<&str>) -> Duration {////// **只搜页面,不搜 data source。** 后者是表格的容器,它自己没有正文;/// 表格里的每一行是一个页面,会在同一次搜索里出现。-pub async fn fetch(token: &str, query: Option<&str>) -> anyhow::Result<(Vec<NotionPage>, bool)> {+///+/// 第三项是正文没读出来的页,每页一句原因。它们不在第一项里,见 `fetch_pages`。+pub async fn fetch(+ token: &str,+ query: Option<&str>,+) -> anyhow::Result<(Vec<NotionPage>, bool, Vec<String>)> {let mut http = Paced::new(client(token)?);fetch_pages(&mut http, query).await}@@ -175,8 +180,9 @@ pub async fn fetch(token: &str, query: Option<&str>) -> anyhow::Result<(Vec<Notiasync fn fetch_pages(http: &mut Paced,query: Option<&str>,-) -> anyhow::Result<(Vec<NotionPage>, bool)> {+) -> anyhow::Result<(Vec<NotionPage>, bool, Vec<String>)> {let mut out = Vec::new();+ let mut unread: Vec<String> = Vec::new();let mut cursor: Option<String> = None;let mut truncated = false;@@ -206,10 +212,18 @@ async fn fetch_pages(}let Some(id) = p["id"].as_str() else { continue };let title = page_title(&p);- let text = page_text(http, id).await.unwrap_or_else(|e| {- tracing::warn!(%id, error = %e, "notion page body could not be read, keeping the title only");- String::new()- });+ // 正文读不出来的页整页跳过:不摄入,库里已有的正文原样留着,不拿仅标题的+ // 版本去替换它。**其余的页照常同步**——中止整轮是另一头的错:有的页永远+ // 读不出来(嵌套块没共享给 integration,Notion 回 404),那样整个来源+ // 就再也不动了。对象存储的目录占位符是同一个教训(#214)。+ let text = match page_text(http, id).await {+ Ok(text) => text,+ Err(error) => {+ tracing::warn!(%id, error = %format!("{error:#}"), "notion page body could not be read, leaving the stored page as it is");+ unread.push(format!("notion page {id}: {error:#}"));+ continue;+ }+ };out.push(NotionPage {external_key: format!("notion://{id}"),@@ -230,7 +244,7 @@ async fn fetch_pages(break;}}- Ok((out, truncated))+ Ok((out, truncated, unread))}/// 页面标题。@@ -307,7 +321,13 @@ async fn append_children(let id = b["id"].as_str().context("notion blocks: child block is missing id")?;- Box::pin(append_children(http, id, out, block_count)).await?;+ if b["type"].as_str() == Some("table") {+ let mut rows = String::new();+ Box::pin(append_children(http, id, &mut rows, block_count)).await?;+ out.push_str(&render_table(&b["table"], &rows)?);+ } else {+ Box::pin(append_children(http, id, out, block_count)).await?;+ }}}if v["has_more"].as_bool() != Some(true) {@@ -324,8 +344,8 @@ async fn append_children(/// 把一个 block 渲染成一行文本。////// **认不出的类型返回它的纯文本而不是丢掉。** Notion 的 block 类型一直在加,-/// 硬编码一张白名单意味着新类型静默消失;而所有带文字的 block 都把文字放在-/// `{type}.rich_text` 下,这个形状很稳。+/// 硬编码一张白名单意味着新类型静默消失;普通文字在 `{type}.rich_text`,+/// 表格行则按 `table_row.cells` 保留每个单元格。fn render_block(b: &serde_json::Value) -> Option<String> {let t = b["type"].as_str()?;let inner = &b[t];@@ -346,6 +366,20 @@ fn render_block(b: &serde_json::Value) -> Option<String> {let lang = inner["language"].as_str().unwrap_or("");format!("```{lang}\n{text}\n```")}+ "table_row" => {+ let cells = inner["cells"].as_array()?;+ let cells: Vec<String> = cells+ .iter()+ .map(|cell| {+ rich_text(cell)+ .replace('\\', "\\\\")+ .replace('|', "\\|")+ .replace("\r\n", "\n")+ .replace(['\r', '\n'], "<br>")+ })+ .collect();+ format!("| {} |", cells.join(" | "))+ }// 分割线与图片没有 rich_text,但它们在正文里也没有信息量"divider" | "image" | "video" | "file" => return None,// child_page 的标题在 `title` 而不是 rich_text@@ -355,6 +389,34 @@ fn render_block(b: &serde_json::Value) -> Option<String> {})}+/// 无列头的表用空表头占位,不能把第一条数据误标为列头。+fn render_table(table: &serde_json::Value, rows: &str) -> anyhow::Result<String> {+ if rows.is_empty() {+ return Ok(String::new());+ }+ let width = table["table_width"]+ .as_u64()+ .context("notion table: missing column count")?;+ let width = usize::try_from(width)?;+ anyhow::ensure!(width > 0, "notion table: column count must be positive");+ let has_header = table["has_column_header"].as_bool() == Some(true);+ let separator = format!("|{}\n", " --- |".repeat(width));+ let mut out = String::from("\n");+ if !has_header {+ out.push_str(&format!("|{}\n", " |".repeat(width)));+ out.push_str(&separator);+ }+ for (index, row) in rows.lines().enumerate() {+ out.push_str(row);+ out.push('\n');+ if index == 0 && has_header {+ out.push_str(&separator);+ }+ }+ out.push('\n');+ Ok(out)+}+/// rich_text 数组拼成纯文本。fn rich_text(v: &serde_json::Value) -> String {v.as_array()@@ -471,7 +533,7 @@ mod tests {eprintln!("跳过:未设 UTOPIA_NOTION_TEST_TOKEN");return Ok(());};- let (pages, _) = fetch(&token, None).await?;+ let (pages, _, _) = fetch(&token, None).await?;assert!(!pages.is_empty(),"一页都没有——integration 可能没有被分享任何页面"diff --git a/crates/utopia-server/src/notion_resync_tests.rs b/crates/utopia-server/src/notion_resync_tests.rsindex 6f350eae..7fe9051e 100644--- a/crates/utopia-server/src/notion_resync_tests.rs+++ b/crates/utopia-server/src/notion_resync_tests.rs@@ -1,5 +1,6 @@//! A page imported before nested traversal must gain its missing body on resync,-//! even when Notion's edit timestamp has not changed.+//! even when Notion's edit timestamp has not changed. A later failed read must+//! leave that complete body and its processing history unchanged.use super::{fetch_pages, Paced};use crate::ingest_sources::{ingest_item, IngestAction};use chrono::{DateTime, Utc};@@ -10,7 +11,7 @@ use uuid::Uuid;use wiremock::{matchers::method, matchers::path, Mock, MockServer, ResponseTemplate};#[tokio::test]-async fn resync_adds_nested_body_without_a_new_edit_timestamp() -> anyhow::Result<()> {+async fn resync_adds_nested_body_and_preserves_it_when_a_later_read_fails() -> anyhow::Result<()> {let Some(url) = utopia_store::test_db::url() else {return Ok(());};@@ -85,7 +86,7 @@ async fn resync_adds_nested_body_without_a_new_edit_timestamp() -> anyhow::Resul}],"has_more": false})))- .expect(2)+ .expect(3).mount(&server).await;Mock::given(method("GET"))@@ -97,7 +98,7 @@ async fn resync_adds_nested_body_without_a_new_edit_timestamp() -> anyhow::Resul}],"has_more": false})))- .expect(2)+ .expect(3).mount(&server).await;Mock::given(method("GET"))@@ -116,7 +117,7 @@ async fn resync_adds_nested_body_without_a_new_edit_timestamp() -> anyhow::Resulhttp.api_root = server.uri();for expected in [IngestAction::Updated, IngestAction::Unchanged] {- let (pages, truncated) = fetch_pages(&mut http, None).await?;+ let (pages, truncated, _) = fetch_pages(&mut http, None).await?;anyhow::ensure!(!truncated && pages.len() == 1);let page = &pages[0];anyhow::ensure!(page.external_key == external_key);@@ -151,9 +152,56 @@ async fn resync_adds_nested_body_without_a_new_edit_timestamp() -> anyhow::Resul.fetch_all(&pool).await?;anyhow::ensure!(- versions == vec![(1, original.sha256.clone()), (2, updated.sha256)],+ versions == vec![(1, original.sha256.clone()), (2, updated.sha256.clone())],"resync must retain the old body and create exactly one new version");+ if expected == IngestAction::Updated {+ let jobs_before: i64 = sqlx::query_scalar(+ "SELECT count(*) FROM jobs WHERE kind='process_document' AND payload->>'document_id'=$1",+ )+ .bind(updated.id.to_string())+ .fetch_one(&pool)+ .await?;+ Mock::given(method("GET"))+ .and(path("/blocks/toggle/children"))+ .respond_with(ResponseTemplate::new(503).set_body_json(json!({+ "message": "temporarily unavailable"+ })))+ .with_priority(1)+ .up_to_n_times(1)+ .expect(1)+ .mount(&server)+ .await;+ let (pages, _, unread) = fetch_pages(&mut http, None).await?;+ anyhow::ensure!(+ pages.is_empty(),+ "a failed body read must not return a page to ingest"+ );+ anyhow::ensure!(+ unread+ == vec!["notion page page: notion blocks returned 503 Service Unavailable: temporarily unavailable".to_string()]+ );+ let retained = documents::find_by_external_key(&pool, source, external_key)+ .await?+ .ok_or_else(|| anyhow::anyhow!("the complete page was lost after a failed read"))?;+ anyhow::ensure!(retained.sha256 == updated.sha256);+ anyhow::ensure!(retained.updated_at == updated.updated_at);+ anyhow::ensure!(state.blob.get(&retained.sha256).await? == complete_body.as_bytes());+ let versions_after: Vec<(i32, String)> = sqlx::query_as(+ "SELECT version, sha256 FROM document_versions WHERE document_id=$1 ORDER BY version",+ )+ .bind(updated.id)+ .fetch_all(&pool)+ .await?;+ anyhow::ensure!(versions_after == versions);+ let jobs_after: i64 = sqlx::query_scalar(+ "SELECT count(*) FROM jobs WHERE kind='process_document' AND payload->>'document_id'=$1",+ )+ .bind(updated.id.to_string())+ .fetch_one(&pool)+ .await?;+ anyhow::ensure!(jobs_after == jobs_before);+ }}Ok::<_, anyhow::Error>(())}diff --git a/crates/utopia-server/src/notion_tests.rs b/crates/utopia-server/src/notion_tests.rsindex 486fe4d1..12b45f92 100644--- a/crates/utopia-server/src/notion_tests.rs+++ b/crates/utopia-server/src/notion_tests.rs@@ -424,12 +424,15 @@ async fn child_rate_limit_exhaustion_is_an_error() {}#[tokio::test]-async fn child_failure_keeps_only_title_and_preserves_page_identity() {+async fn a_page_whose_body_cannot_be_read_is_left_out_and_the_others_still_come_back() {let server = MockServer::start().await;+ let page = |id: &str, title: &str| {+ json!({"id": id, "properties": {"Name": {"type": "title", "title": [{"plain_text": title}]}},+ "last_edited_time": "2026-10-01T12:00:00Z"})+ };Mock::given(method("POST")).and(path("/search")).respond_with(ResponseTemplate::new(200).set_body_json(json!({- "results":[{"id":"page", "properties":{"Name":{"type":"title", "title":[{"plain_text":"Quarterly policy"}]}},- "last_edited_time":"2026-10-01T12:00:00Z"}], "has_more":false+ "results": [page("page", "Quarterly policy"), page("other", "Travel policy")], "has_more": false}))).expect(1).mount(&server).await;children(&server,@@ -440,18 +443,137 @@ async fn child_failure_keeps_only_title_and_preserves_page_identity() {).await;Mock::given(path("/blocks/child/children"))- .respond_with(ResponseTemplate::new(500).set_body_json(json!({"message":"unavailable"})))+ .respond_with(ResponseTemplate::new(503).set_body_json(json!({"message":"unavailable"}))).expect(1).mount(&server).await;- let (pages, truncated) = fetch_pages(&mut paced(&server), None).await.unwrap();- assert!(!truncated);+ children(+ &server,+ "other",+ None,+ vec![block(+ "p",+ "paragraph",+ "Economy class under six hours",+ false,+ )],+ None,+ )+ .await;++ let (pages, truncated, unread) = fetch_pages(&mut paced(&server), None).await.unwrap();+ // 读坏的那一页不带着半截正文或只有标题回来,另一页不受它连累assert_eq!(pages.len(), 1);- assert_eq!(pages[0].external_key, "notion://page");- assert_eq!(pages[0].filename, "Quarterly-policy.md");+ assert_eq!(pages[0].external_key, "notion://other");+ assert!(pages[0].text.contains("Economy class under six hours"));+ assert!(!truncated);assert_eq!(- pages[0].last_edited,- Some("2026-10-01T12:00:00Z".parse().unwrap())+ unread,+ vec!["notion page page: notion blocks returned 503 Service Unavailable: unavailable"]+ );+}++#[tokio::test]+async fn a_nested_table_keeps_its_header_cells_and_paginated_rows() {+ let server = MockServer::start().await;+ children(+ &server,+ "page",+ None,+ vec![block("toggle", "toggle", "Quarterly policy", true)],+ None,+ )+ .await;+ children(+ &server,+ "toggle",+ None,+ vec![+ json!({"id":"table", "type":"table", "has_children":true,+ "table":{"table_width":3, "has_column_header":true, "has_row_header":true}}),+ block("after", "paragraph", "After table", false),+ ],+ None,+ )+ .await;+ children(+ &server,+ "table",+ None,+ vec![+ json!({"type":"table_row", "table_row":{"cells":[+ [{"plain_text":"Metric"}], [{"plain_text":"Value"}], [{"plain_text":"Notes"}]+ ]}}),+ json!({"type":"table_row", "table_row":{"cells":[+ [{"plain_text":"Revenue"}], [], [{"plain_text":"Pending"}]+ ]}}),+ ],+ Some("table-next"),+ )+ .await;+ children(+ &server,+ "table",+ Some("table-next"),+ vec![json!({"type":"table_row", "table_row":{"cells":[+ [{"plain_text":"Pipe | slash \\"}],+ [{"plain_text":"42"}, {"plain_text":" million"}],+ [{"plain_text":"First\r\nSecond\nThird\rLast"}]+ ]}})],+ None,+ )+ .await;+ let text = page_text(&mut paced(&server), "page").await.unwrap();+ assert_eq!(+ text,+ "Quarterly policy\n\n| Metric | Value | Notes |\n| --- | --- | --- |\n| Revenue | | Pending |\n| Pipe \\| slash \\\\ | 42 million | First<br>Second<br>Third<br>Last |\n\nAfter table\n"+ );+ assert_eq!(+ request_paths(&server).await,+ [+ "/blocks/page/children",+ "/blocks/toggle/children",+ "/blocks/table/children",+ "/blocks/table/children"+ ]+ );+ let chunks = utopia_ingest::chunk_text(&text);+ assert!(chunks+ .iter()+ .any(|chunk| chunk.text.contains("| --- | --- | --- |")));+}++#[tokio::test]+async fn a_table_without_column_headers_keeps_the_first_row_as_data() {+ let server = MockServer::start().await;+ children(+ &server,+ "page",+ None,+ vec![+ block("before", "paragraph", "Before table", false),+ json!({"id":"table", "type":"table", "has_children":true,+ "table":{"table_width":2, "has_column_header":false}}),+ block("after", "paragraph", "After table", false),+ ],+ None,+ )+ .await;+ children(+ &server,+ "table",+ None,+ vec![+ json!({"type":"table_row", "table_row":{"cells":[+ [{"plain_text":"Revenue"}], [{"plain_text":"42 million"}]+ ]}}),+ json!({"type":"table_row", "table_row":{"cells":[[], []]}}),+ ],+ None,+ )+ .await;+ assert_eq!(+ page_text(&mut paced(&server), "page").await.unwrap(),+ "Before table\n\n| | |\n| --- | --- |\n| Revenue | 42 million |\n| | |\n\nAfter table\n");- assert_eq!(pages[0].text, "# Quarterly policy\n\n");}