Skip to content

World's first open-source enterprise world model.

Preserve Notion page bodies on read failures and import table cells (#1101)

Daniel654141aa1c57
diff --git a/crates/utopia-server/src/ingest_sources.rs b/crates/utopia-server/src/ingest_sources.rs
index b0cedc1d..62f81c69 100644
--- a/crates/utopia-server/src/ingest_sources.rs
+++ b/crates/utopia-server/src/ingest_sources.rs
@@ -1165,7 +1165,7 @@ async fn sync_notion(state: &AppState, source: &Source) -> anyhow::Result<SyncSt
.map(str::trim)
.filter(|s| !s.is_empty());
- let (pages, truncated) = crate::notion::fetch(token, query).await?;
+ let (pages, truncated, unread) = crate::notion::fetch(token, query).await?;
if truncated {
tracing::warn!("页面数到达单次上限,其余留给下一次同步");
}
@@ -1185,6 +1185,21 @@ async fn sync_notion(state: &AppState, source: &Source) -> anyhow::Result<SyncSt
.await?;
stats.absorb(action);
}
+ // 读得出来的页已经进库了;读不出来的那几页让这一轮记成失败并点名,
+ // 不然它们停在旧版本上没有人知道
+ if !unread.is_empty() {
+ anyhow::bail!(
+ "{} Notion page(s) synced; {} could not be read and keep their stored content: {}",
+ stats.total(),
+ unread.len(),
+ unread
+ .iter()
+ .take(3)
+ .map(String::as_str)
+ .collect::<Vec<_>>()
+ .join("; ")
+ );
+ }
Ok(stats)
}
diff --git a/crates/utopia-server/src/notion.rs b/crates/utopia-server/src/notion.rs
index 344e3a54..e53f1390 100644
--- a/crates/utopia-server/src/notion.rs
+++ b/crates/utopia-server/src/notion.rs
@@ -167,7 +167,12 @@ fn retry_after(header: Option<&str>) -> Duration {
///
/// **只搜页面,不搜 data source。** 后者是表格的容器,它自己没有正文;
/// 表格里的每一行是一个页面,会在同一次搜索里出现。
-pub async fn fetch(token: &str, query: Option<&str>) -> anyhow::Result<(Vec<NotionPage>, bool)> {
+///
+/// 第三项是正文没读出来的页,每页一句原因。它们不在第一项里,见 `fetch_pages`。
+pub async fn fetch(
+ token: &str,
+ query: Option<&str>,
+) -> anyhow::Result<(Vec<NotionPage>, bool, Vec<String>)> {
let mut http = Paced::new(client(token)?);
fetch_pages(&mut http, query).await
}
@@ -175,8 +180,9 @@ pub async fn fetch(token: &str, query: Option<&str>) -> anyhow::Result<(Vec<Noti
async fn fetch_pages(
http: &mut Paced,
query: Option<&str>,
-) -> anyhow::Result<(Vec<NotionPage>, bool)> {
+) -> anyhow::Result<(Vec<NotionPage>, bool, Vec<String>)> {
let mut out = Vec::new();
+ let mut unread: Vec<String> = Vec::new();
let mut cursor: Option<String> = None;
let mut truncated = false;
@@ -206,10 +212,18 @@ async fn fetch_pages(
}
let Some(id) = p["id"].as_str() else { continue };
let title = page_title(&p);
- let text = page_text(http, id).await.unwrap_or_else(|e| {
- tracing::warn!(%id, error = %e, "notion page body could not be read, keeping the title only");
- String::new()
- });
+ // 正文读不出来的页整页跳过:不摄入,库里已有的正文原样留着,不拿仅标题的
+ // 版本去替换它。**其余的页照常同步**——中止整轮是另一头的错:有的页永远
+ // 读不出来(嵌套块没共享给 integration,Notion 回 404),那样整个来源
+ // 就再也不动了。对象存储的目录占位符是同一个教训(#214)。
+ let text = match page_text(http, id).await {
+ Ok(text) => text,
+ Err(error) => {
+ tracing::warn!(%id, error = %format!("{error:#}"), "notion page body could not be read, leaving the stored page as it is");
+ unread.push(format!("notion page {id}: {error:#}"));
+ continue;
+ }
+ };
out.push(NotionPage {
external_key: format!("notion://{id}"),
@@ -230,7 +244,7 @@ async fn fetch_pages(
break;
}
}
- Ok((out, truncated))
+ Ok((out, truncated, unread))
}
/// 页面标题。
@@ -307,7 +321,13 @@ async fn append_children(
let id = b["id"]
.as_str()
.context("notion blocks: child block is missing id")?;
- Box::pin(append_children(http, id, out, block_count)).await?;
+ if b["type"].as_str() == Some("table") {
+ let mut rows = String::new();
+ Box::pin(append_children(http, id, &mut rows, block_count)).await?;
+ out.push_str(&render_table(&b["table"], &rows)?);
+ } else {
+ Box::pin(append_children(http, id, out, block_count)).await?;
+ }
}
}
if v["has_more"].as_bool() != Some(true) {
@@ -324,8 +344,8 @@ async fn append_children(
/// 把一个 block 渲染成一行文本。
///
/// **认不出的类型返回它的纯文本而不是丢掉。** Notion 的 block 类型一直在加,
-/// 硬编码一张白名单意味着新类型静默消失;而所有带文字的 block 都把文字放在
-/// `{type}.rich_text` 下,这个形状很稳。
+/// 硬编码一张白名单意味着新类型静默消失;普通文字在 `{type}.rich_text`,
+/// 表格行则按 `table_row.cells` 保留每个单元格。
fn render_block(b: &serde_json::Value) -> Option<String> {
let t = b["type"].as_str()?;
let inner = &b[t];
@@ -346,6 +366,20 @@ fn render_block(b: &serde_json::Value) -> Option<String> {
let lang = inner["language"].as_str().unwrap_or("");
format!("```{lang}\n{text}\n```")
}
+ "table_row" => {
+ let cells = inner["cells"].as_array()?;
+ let cells: Vec<String> = cells
+ .iter()
+ .map(|cell| {
+ rich_text(cell)
+ .replace('\\', "\\\\")
+ .replace('|', "\\|")
+ .replace("\r\n", "\n")
+ .replace(['\r', '\n'], "<br>")
+ })
+ .collect();
+ format!("| {} |", cells.join(" | "))
+ }
// 分割线与图片没有 rich_text,但它们在正文里也没有信息量
"divider" | "image" | "video" | "file" => return None,
// child_page 的标题在 `title` 而不是 rich_text
@@ -355,6 +389,34 @@ fn render_block(b: &serde_json::Value) -> Option<String> {
})
}
+/// 无列头的表用空表头占位,不能把第一条数据误标为列头。
+fn render_table(table: &serde_json::Value, rows: &str) -> anyhow::Result<String> {
+ if rows.is_empty() {
+ return Ok(String::new());
+ }
+ let width = table["table_width"]
+ .as_u64()
+ .context("notion table: missing column count")?;
+ let width = usize::try_from(width)?;
+ anyhow::ensure!(width > 0, "notion table: column count must be positive");
+ let has_header = table["has_column_header"].as_bool() == Some(true);
+ let separator = format!("|{}\n", " --- |".repeat(width));
+ let mut out = String::from("\n");
+ if !has_header {
+ out.push_str(&format!("|{}\n", " |".repeat(width)));
+ out.push_str(&separator);
+ }
+ for (index, row) in rows.lines().enumerate() {
+ out.push_str(row);
+ out.push('\n');
+ if index == 0 && has_header {
+ out.push_str(&separator);
+ }
+ }
+ out.push('\n');
+ Ok(out)
+}
+
/// rich_text 数组拼成纯文本。
fn rich_text(v: &serde_json::Value) -> String {
v.as_array()
@@ -471,7 +533,7 @@ mod tests {
eprintln!("跳过:未设 UTOPIA_NOTION_TEST_TOKEN");
return Ok(());
};
- let (pages, _) = fetch(&token, None).await?;
+ let (pages, _, _) = fetch(&token, None).await?;
assert!(
!pages.is_empty(),
"一页都没有——integration 可能没有被分享任何页面"
diff --git a/crates/utopia-server/src/notion_resync_tests.rs b/crates/utopia-server/src/notion_resync_tests.rs
index 6f350eae..7fe9051e 100644
--- a/crates/utopia-server/src/notion_resync_tests.rs
+++ b/crates/utopia-server/src/notion_resync_tests.rs
@@ -1,5 +1,6 @@
//! A page imported before nested traversal must gain its missing body on resync,
-//! even when Notion's edit timestamp has not changed.
+//! even when Notion's edit timestamp has not changed. A later failed read must
+//! leave that complete body and its processing history unchanged.
use super::{fetch_pages, Paced};
use crate::ingest_sources::{ingest_item, IngestAction};
use chrono::{DateTime, Utc};
@@ -10,7 +11,7 @@ use uuid::Uuid;
use wiremock::{matchers::method, matchers::path, Mock, MockServer, ResponseTemplate};
#[tokio::test]
-async fn resync_adds_nested_body_without_a_new_edit_timestamp() -> anyhow::Result<()> {
+async fn resync_adds_nested_body_and_preserves_it_when_a_later_read_fails() -> anyhow::Result<()> {
let Some(url) = utopia_store::test_db::url() else {
return Ok(());
};
@@ -85,7 +86,7 @@ async fn resync_adds_nested_body_without_a_new_edit_timestamp() -> anyhow::Resul
}],
"has_more": false
})))
- .expect(2)
+ .expect(3)
.mount(&server)
.await;
Mock::given(method("GET"))
@@ -97,7 +98,7 @@ async fn resync_adds_nested_body_without_a_new_edit_timestamp() -> anyhow::Resul
}],
"has_more": false
})))
- .expect(2)
+ .expect(3)
.mount(&server)
.await;
Mock::given(method("GET"))
@@ -116,7 +117,7 @@ async fn resync_adds_nested_body_without_a_new_edit_timestamp() -> anyhow::Resul
http.api_root = server.uri();
for expected in [IngestAction::Updated, IngestAction::Unchanged] {
- let (pages, truncated) = fetch_pages(&mut http, None).await?;
+ let (pages, truncated, _) = fetch_pages(&mut http, None).await?;
anyhow::ensure!(!truncated && pages.len() == 1);
let page = &pages[0];
anyhow::ensure!(page.external_key == external_key);
@@ -151,9 +152,56 @@ async fn resync_adds_nested_body_without_a_new_edit_timestamp() -> anyhow::Resul
.fetch_all(&pool)
.await?;
anyhow::ensure!(
- versions == vec![(1, original.sha256.clone()), (2, updated.sha256)],
+ versions == vec![(1, original.sha256.clone()), (2, updated.sha256.clone())],
"resync must retain the old body and create exactly one new version"
);
+ if expected == IngestAction::Updated {
+ let jobs_before: i64 = sqlx::query_scalar(
+ "SELECT count(*) FROM jobs WHERE kind='process_document' AND payload->>'document_id'=$1",
+ )
+ .bind(updated.id.to_string())
+ .fetch_one(&pool)
+ .await?;
+ Mock::given(method("GET"))
+ .and(path("/blocks/toggle/children"))
+ .respond_with(ResponseTemplate::new(503).set_body_json(json!({
+ "message": "temporarily unavailable"
+ })))
+ .with_priority(1)
+ .up_to_n_times(1)
+ .expect(1)
+ .mount(&server)
+ .await;
+ let (pages, _, unread) = fetch_pages(&mut http, None).await?;
+ anyhow::ensure!(
+ pages.is_empty(),
+ "a failed body read must not return a page to ingest"
+ );
+ anyhow::ensure!(
+ unread
+ == vec!["notion page page: notion blocks returned 503 Service Unavailable: temporarily unavailable".to_string()]
+ );
+ let retained = documents::find_by_external_key(&pool, source, external_key)
+ .await?
+ .ok_or_else(|| anyhow::anyhow!("the complete page was lost after a failed read"))?;
+ anyhow::ensure!(retained.sha256 == updated.sha256);
+ anyhow::ensure!(retained.updated_at == updated.updated_at);
+ anyhow::ensure!(state.blob.get(&retained.sha256).await? == complete_body.as_bytes());
+ let versions_after: Vec<(i32, String)> = sqlx::query_as(
+ "SELECT version, sha256 FROM document_versions WHERE document_id=$1 ORDER BY version",
+ )
+ .bind(updated.id)
+ .fetch_all(&pool)
+ .await?;
+ anyhow::ensure!(versions_after == versions);
+ let jobs_after: i64 = sqlx::query_scalar(
+ "SELECT count(*) FROM jobs WHERE kind='process_document' AND payload->>'document_id'=$1",
+ )
+ .bind(updated.id.to_string())
+ .fetch_one(&pool)
+ .await?;
+ anyhow::ensure!(jobs_after == jobs_before);
+ }
}
Ok::<_, anyhow::Error>(())
}
diff --git a/crates/utopia-server/src/notion_tests.rs b/crates/utopia-server/src/notion_tests.rs
index 486fe4d1..12b45f92 100644
--- a/crates/utopia-server/src/notion_tests.rs
+++ b/crates/utopia-server/src/notion_tests.rs
@@ -424,12 +424,15 @@ async fn child_rate_limit_exhaustion_is_an_error() {
}
#[tokio::test]
-async fn child_failure_keeps_only_title_and_preserves_page_identity() {
+async fn a_page_whose_body_cannot_be_read_is_left_out_and_the_others_still_come_back() {
let server = MockServer::start().await;
+ let page = |id: &str, title: &str| {
+ json!({"id": id, "properties": {"Name": {"type": "title", "title": [{"plain_text": title}]}},
+ "last_edited_time": "2026-10-01T12:00:00Z"})
+ };
Mock::given(method("POST")).and(path("/search"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
- "results":[{"id":"page", "properties":{"Name":{"type":"title", "title":[{"plain_text":"Quarterly policy"}]}},
- "last_edited_time":"2026-10-01T12:00:00Z"}], "has_more":false
+ "results": [page("page", "Quarterly policy"), page("other", "Travel policy")], "has_more": false
}))).expect(1).mount(&server).await;
children(
&server,
@@ -440,18 +443,137 @@ async fn child_failure_keeps_only_title_and_preserves_page_identity() {
)
.await;
Mock::given(path("/blocks/child/children"))
- .respond_with(ResponseTemplate::new(500).set_body_json(json!({"message":"unavailable"})))
+ .respond_with(ResponseTemplate::new(503).set_body_json(json!({"message":"unavailable"})))
.expect(1)
.mount(&server)
.await;
- let (pages, truncated) = fetch_pages(&mut paced(&server), None).await.unwrap();
- assert!(!truncated);
+ children(
+ &server,
+ "other",
+ None,
+ vec![block(
+ "p",
+ "paragraph",
+ "Economy class under six hours",
+ false,
+ )],
+ None,
+ )
+ .await;
+
+ let (pages, truncated, unread) = fetch_pages(&mut paced(&server), None).await.unwrap();
+ // 读坏的那一页不带着半截正文或只有标题回来,另一页不受它连累
assert_eq!(pages.len(), 1);
- assert_eq!(pages[0].external_key, "notion://page");
- assert_eq!(pages[0].filename, "Quarterly-policy.md");
+ assert_eq!(pages[0].external_key, "notion://other");
+ assert!(pages[0].text.contains("Economy class under six hours"));
+ assert!(!truncated);
assert_eq!(
- pages[0].last_edited,
- Some("2026-10-01T12:00:00Z".parse().unwrap())
+ unread,
+ vec!["notion page page: notion blocks returned 503 Service Unavailable: unavailable"]
+ );
+}
+
+#[tokio::test]
+async fn a_nested_table_keeps_its_header_cells_and_paginated_rows() {
+ let server = MockServer::start().await;
+ children(
+ &server,
+ "page",
+ None,
+ vec![block("toggle", "toggle", "Quarterly policy", true)],
+ None,
+ )
+ .await;
+ children(
+ &server,
+ "toggle",
+ None,
+ vec![
+ json!({"id":"table", "type":"table", "has_children":true,
+ "table":{"table_width":3, "has_column_header":true, "has_row_header":true}}),
+ block("after", "paragraph", "After table", false),
+ ],
+ None,
+ )
+ .await;
+ children(
+ &server,
+ "table",
+ None,
+ vec![
+ json!({"type":"table_row", "table_row":{"cells":[
+ [{"plain_text":"Metric"}], [{"plain_text":"Value"}], [{"plain_text":"Notes"}]
+ ]}}),
+ json!({"type":"table_row", "table_row":{"cells":[
+ [{"plain_text":"Revenue"}], [], [{"plain_text":"Pending"}]
+ ]}}),
+ ],
+ Some("table-next"),
+ )
+ .await;
+ children(
+ &server,
+ "table",
+ Some("table-next"),
+ vec![json!({"type":"table_row", "table_row":{"cells":[
+ [{"plain_text":"Pipe | slash \\"}],
+ [{"plain_text":"42"}, {"plain_text":" million"}],
+ [{"plain_text":"First\r\nSecond\nThird\rLast"}]
+ ]}})],
+ None,
+ )
+ .await;
+ let text = page_text(&mut paced(&server), "page").await.unwrap();
+ assert_eq!(
+ text,
+ "Quarterly policy\n\n| Metric | Value | Notes |\n| --- | --- | --- |\n| Revenue | | Pending |\n| Pipe \\| slash \\\\ | 42 million | First<br>Second<br>Third<br>Last |\n\nAfter table\n"
+ );
+ assert_eq!(
+ request_paths(&server).await,
+ [
+ "/blocks/page/children",
+ "/blocks/toggle/children",
+ "/blocks/table/children",
+ "/blocks/table/children"
+ ]
+ );
+ let chunks = utopia_ingest::chunk_text(&text);
+ assert!(chunks
+ .iter()
+ .any(|chunk| chunk.text.contains("| --- | --- | --- |")));
+}
+
+#[tokio::test]
+async fn a_table_without_column_headers_keeps_the_first_row_as_data() {
+ let server = MockServer::start().await;
+ children(
+ &server,
+ "page",
+ None,
+ vec![
+ block("before", "paragraph", "Before table", false),
+ json!({"id":"table", "type":"table", "has_children":true,
+ "table":{"table_width":2, "has_column_header":false}}),
+ block("after", "paragraph", "After table", false),
+ ],
+ None,
+ )
+ .await;
+ children(
+ &server,
+ "table",
+ None,
+ vec![
+ json!({"type":"table_row", "table_row":{"cells":[
+ [{"plain_text":"Revenue"}], [{"plain_text":"42 million"}]
+ ]}}),
+ json!({"type":"table_row", "table_row":{"cells":[[], []]}}),
+ ],
+ None,
+ )
+ .await;
+ assert_eq!(
+ page_text(&mut paced(&server), "page").await.unwrap(),
+ "Before table\n\n| | |\n| --- | --- |\n| Revenue | 42 million |\n| | |\n\nAfter table\n"
);
- assert_eq!(pages[0].text, "# Quarterly policy\n\n");
}