World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
A Notion page keeps its own clock, and the title hides under a type (#213)
WaylandYang0f3ff5af92bd
diff --git a/crates/utopia-server/src/ingest_sources.rs b/crates/utopia-server/src/ingest_sources.rsindex 936c0e87..56afd33c 100644--- a/crates/utopia-server/src/ingest_sources.rs+++ b/crates/utopia-server/src/ingest_sources.rs@@ -56,6 +56,7 @@ pub async fn sync_source(state: &AppState, source_id: Uuid) -> anyhow::Result<()"jira_issues" => sync_jira_issues(state, &source).await,"s3" | "azure_blob" | "gcs" => sync_object_storage(state, &source).await,"webdav" => sync_webdav(state, &source).await,+ "notion" => sync_notion(state, &source).await,// folder / api 无拉取语义_ => Ok(SyncStats::default()),};@@ -850,6 +851,45 @@ async fn sync_webdav(state: &AppState, source: &Source) -> anyhow::Result<SyncStOk(stats)}+/// Notion 同步:把 integration 能看见的页面摄进来。+///+/// `doc_time` 取 `last_edited_time`——**这是页面自己的编辑时刻**,比对象存储+/// 那边的写入时刻实在:一份 2019 年的合同今天传进 S3 会落在今天,而 Notion+/// 页面的编辑时刻就是它内容变化的时刻。+async fn sync_notion(state: &AppState, source: &Source) -> anyhow::Result<SyncStats> {+ let token = source.config["token"]+ .as_str()+ .map(str::trim)+ .filter(|s| !s.is_empty())+ .ok_or_else(|| anyhow::anyhow!("notion source is missing config.token"))?;+ let query = source.config["query"]+ .as_str()+ .map(str::trim)+ .filter(|s| !s.is_empty());++ let (pages, truncated) = crate::notion::fetch(token, query).await?;+ if truncated {+ tracing::warn!("页面数到达单次上限,其余留给下一次同步");+ }++ let mut stats = SyncStats::default();+ for p in pages {+ let action = ingest_item(+ state,+ source.kb_id,+ source.id,+ &p.external_key,+ &p.filename,+ "text/markdown",+ p.text.as_bytes(),+ p.last_edited,+ )+ .await?;+ stats.absorb(action);+ }+ Ok(stats)+}+/// 标题无扩展名时按 mime 补一个,接入解析矩阵的分派。fn ensure_extension(title: &str, mime: &str) -> String {let has_ext = titlediff --git a/crates/utopia-server/src/main.rs b/crates/utopia-server/src/main.rsindex 8cf85a8c..d23cabeb 100644--- a/crates/utopia-server/src/main.rs+++ b/crates/utopia-server/src/main.rs@@ -14,6 +14,7 @@ mod jira_issues;mod live;mod llm_util;mod mappings;+mod notion;mod object_storage;mod ontology_index;mod ontology_packs;diff --git a/crates/utopia-server/src/notion.rs b/crates/utopia-server/src/notion.rsnew file mode 100644index 00000000..e37bb835--- /dev/null+++ b/crates/utopia-server/src/notion.rs@@ -0,0 +1,350 @@+//! Notion 来源:把 integration 能看见的页面同步进来。+//!+//! 按 [0013](../../docs/decisions/0013-a-source-should-hand-over-its-history.md)+//! 的四条判据,它比对象存储强一档:+//!+//! | 判据 | Notion |+//! |---|---|+//! | 真实时间戳 | `last_edited_time`,**是文档自己的编辑时刻**,不是我们抓它的时刻 |+//! | 会不会自我推翻 | 页面被反复改写正是它的常态 |+//! | 稳定身份 | 页面 UUID,改标题、挪位置都不变 |+//! | 企业知识住不住在那儿 | 制度、会议纪要、决策记录——正是这套系统要的东西 |+//!+//! **但它只交出现状,不交出历史。** Notion 的版本历史不在公开 API 里,+//! 所以跟工单系统不同:一次同步只能看见此刻,之前的编辑全靠一次次同步慢慢攒。+//! 这跟 `url` / `rss` 是同一个形状,而工单那两个能一次把变更史拿全。+//!+//! ## 两个容易踩的+//!+//! **`Notion-Version` 头是必填的**,而且值是日期。少了它接口直接 400,+//! 而错误信息只说 "missing version",不会告诉你该填哪个。+//!+//! **限流是每秒三次**(官方说法是「平均三次」)。所以取页面内容时是顺序而不是+//! 并发——并发上去只会换来一串 429,而我们的重试退避是为抽取那条路设计的,+//! 不该被摄入路径借用。++use anyhow::Context as _;+use chrono::{DateTime, Utc};++/// 请求头里的 API 版本。**写死而不是留给配置**:响应的形状跟着它变,+/// 让用户填一个我们没适配过的版本,换来的是解析静默失配。+const NOTION_VERSION: &str = "2026-03-11";++/// 一次同步最多取多少页。理由同别的来源:摄入不可逆,而一个 workspace+/// 可以有几万页。+const MAX_PAGES_PER_SYNC: usize = 500;++/// 每页最多取多少个 block。再深的页面截断,比让一次同步卡在一页上好。+const MAX_BLOCKS_PER_PAGE: usize = 500;++/// 一个待摄入的页面。+pub struct NotionPage {+ /// `notion://{page_id}`——页面 UUID 是它最稳的身份+ pub external_key: String,+ pub filename: String,+ pub text: String,+ pub last_edited: Option<DateTime<Utc>>,+}++fn client(token: &str) -> anyhow::Result<reqwest::Client> {+ let mut h = reqwest::header::HeaderMap::new();+ h.insert("Notion-Version", NOTION_VERSION.parse()?);+ h.insert(+ reqwest::header::AUTHORIZATION,+ format!("Bearer {token}").parse()?,+ );+ Ok(reqwest::Client::builder()+ .default_headers(h)+ .timeout(std::time::Duration::from_secs(60))+ .build()?)+}++/// 取 integration 能看见的所有页面。+///+/// **只搜页面,不搜 data source。** 后者是表格的容器,它自己没有正文;+/// 表格里的每一行是一个页面,会在同一次搜索里出现。+pub async fn fetch(token: &str, query: Option<&str>) -> anyhow::Result<(Vec<NotionPage>, bool)> {+ let http = client(token)?;+ let mut out = Vec::new();+ let mut cursor: Option<String> = None;+ let mut truncated = false;++ loop {+ let mut body = serde_json::json!({+ "filter": { "property": "object", "value": "page" },+ "page_size": 100,+ });+ if let Some(q) = query {+ body["query"] = serde_json::Value::String(q.to_string());+ }+ if let Some(c) = &cursor {+ body["start_cursor"] = serde_json::Value::String(c.clone());+ }++ let resp = http+ .post("https://api.notion.com/v1/search")+ .json(&body)+ .send()+ .await+ .context("notion search")?;+ let status = resp.status();+ let v: serde_json::Value = resp.json().await.context("notion search 响应")?;+ if !status.is_success() {+ anyhow::bail!(+ "notion search 回了 {status}: {}",+ v["message"].as_str().unwrap_or("unknown")+ );+ }++ for p in v["results"].as_array().unwrap_or(&vec![]).clone() {+ // 回收站里的和归档的都不要——它们在界面上已经不算数了+ if p["in_trash"].as_bool() == Some(true) || p["is_archived"].as_bool() == Some(true) {+ continue;+ }+ if out.len() >= MAX_PAGES_PER_SYNC {+ truncated = true;+ break;+ }+ let Some(id) = p["id"].as_str() else { continue };+ let title = page_title(&p);+ let text = page_text(&http, id).await.unwrap_or_else(|e| {+ tracing::warn!(%id, error = %e, "页面正文取不回来,只留标题");+ String::new()+ });++ out.push(NotionPage {+ external_key: format!("notion://{id}"),+ filename: format!("{}.md", slug(&title)),+ text: format!("# {title}\n\n{text}"),+ last_edited: p["last_edited_time"]+ .as_str()+ .and_then(|s| DateTime::parse_from_rfc3339(s).ok())+ .map(|d| d.with_timezone(&Utc)),+ });+ }++ if truncated || v["has_more"].as_bool() != Some(true) {+ break;+ }+ cursor = v["next_cursor"].as_str().map(str::to_string);+ if cursor.is_none() {+ break;+ }+ }+ Ok((out, truncated))+}++/// 页面标题。+///+/// **标题藏在 `properties` 里那个 `type == "title"` 的属性下,而它的名字不固定**:+/// 数据库里的页面可能叫 `Name`、`名称`、`任务`,普通页面叫 `title`。+/// 按名字找会在别人的 workspace 上找不到,所以按类型找。+fn page_title(page: &serde_json::Value) -> String {+ let props = page["properties"].as_object();+ let t = props.and_then(|m| {+ m.values()+ .find(|v| v["type"] == "title")+ .and_then(|v| v["title"].as_array())+ });+ let s = t+ .map(|arr| {+ arr.iter()+ .filter_map(|r| r["plain_text"].as_str())+ .collect::<String>()+ })+ .unwrap_or_default();+ if s.trim().is_empty() {+ "untitled".into()+ } else {+ s+ }+}++/// 取一页的正文,逐层展开 block。+async fn page_text(http: &reqwest::Client, page_id: &str) -> anyhow::Result<String> {+ let mut out = String::new();+ let mut n = 0usize;+ let mut cursor: Option<String> = None;++ loop {+ let mut url = format!("https://api.notion.com/v1/blocks/{page_id}/children?page_size=100");+ if let Some(c) = &cursor {+ url.push_str(&format!("&start_cursor={c}"));+ }+ let resp = http.get(&url).send().await?;+ if !resp.status().is_success() {+ anyhow::bail!("blocks 回了 {}", resp.status());+ }+ let v: serde_json::Value = resp.json().await?;++ for b in v["results"].as_array().unwrap_or(&vec![]) {+ if n >= MAX_BLOCKS_PER_PAGE {+ return Ok(out);+ }+ n += 1;+ if let Some(line) = render_block(b) {+ out.push_str(&line);+ out.push('\n');+ }+ }+ if v["has_more"].as_bool() != Some(true) {+ break;+ }+ cursor = v["next_cursor"].as_str().map(str::to_string);+ if cursor.is_none() {+ break;+ }+ }+ Ok(out)+}++/// 把一个 block 渲染成一行文本。+///+/// **认不出的类型返回它的纯文本而不是丢掉。** Notion 的 block 类型一直在加,+/// 硬编码一张白名单意味着新类型静默消失;而所有带文字的 block 都把文字放在+/// `{type}.rich_text` 下,这个形状很稳。+fn render_block(b: &serde_json::Value) -> Option<String> {+ let t = b["type"].as_str()?;+ let inner = &b[t];+ let text = rich_text(&inner["rich_text"]);++ Some(match t {+ "heading_1" => format!("## {text}"),+ "heading_2" => format!("### {text}"),+ "heading_3" => format!("#### {text}"),+ "bulleted_list_item" => format!("- {text}"),+ "numbered_list_item" => format!("1. {text}"),+ "to_do" => {+ let done = inner["checked"].as_bool() == Some(true);+ format!("- [{}] {text}", if done { "x" } else { " " })+ }+ "quote" => format!("> {text}"),+ "code" => {+ let lang = inner["language"].as_str().unwrap_or("");+ format!("```{lang}\n{text}\n```")+ }+ // 分割线与图片没有 rich_text,但它们在正文里也没有信息量+ "divider" | "image" | "video" | "file" => return None,+ // child_page 的标题在 `title` 而不是 rich_text+ "child_page" => format!("- {}", inner["title"].as_str().unwrap_or("")),+ _ if text.trim().is_empty() => return None,+ _ => text,+ })+}++/// rich_text 数组拼成纯文本。+fn rich_text(v: &serde_json::Value) -> String {+ v.as_array()+ .map(|arr| {+ arr.iter()+ .filter_map(|r| r["plain_text"].as_str())+ .collect::<String>()+ })+ .unwrap_or_default()+}++/// 标题变成能当文件名的东西。+fn slug(title: &str) -> String {+ let s: String = title+ .chars()+ .map(|c| if c.is_alphanumeric() { c } else { '-' })+ .collect();+ let s = s.trim_matches('-').to_string();+ if s.is_empty() {+ "untitled".into()+ } else {+ s.chars().take(60).collect()+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ /// **标题属性的名字是任意的。** 数据库里的页面可能把它叫 `Name`、`名称`、+ /// `任务`;按名字找会在别人的 workspace 上返回 untitled,而那看起来+ /// 像是「页面没有标题」而不是「我们找错了地方」。+ #[test]+ fn a_title_is_found_by_type_not_by_name() {+ for key in ["title", "Name", "名称", "任务"] {+ let page = serde_json::json!({+ "properties": {+ key: { "type": "title", "title": [{ "plain_text": "季度复盘" }] },+ "Status": { "type": "select", "select": { "name": "Done" } }+ }+ });+ assert_eq!(page_title(&page), "季度复盘", "属性名 {key} 时找不到标题");+ }+ }++ /// 富文本是分段的——加粗、链接都会把一句话切开。拼不全就会丢字。+ #[test]+ fn rich_text_segments_join_back_into_one_line() {+ let v = serde_json::json!([+ { "plain_text": "把总部搬到" },+ { "plain_text": "深圳" },+ { "plain_text": "了" }+ ]);+ assert_eq!(rich_text(&v), "把总部搬到深圳了");+ }++ /// **认不出的 block 类型不能丢。** Notion 一直在加类型,而带文字的+ /// block 都把文字放在 `{type}.rich_text` 下——按白名单渲染会让新类型+ /// 静默消失。+ #[test]+ fn an_unknown_block_keeps_its_text() {+ let b = serde_json::json!({+ "type": "some_new_block_type_2027",+ "some_new_block_type_2027": { "rich_text": [{ "plain_text": "还是有内容的" }] }+ });+ assert_eq!(render_block(&b).as_deref(), Some("还是有内容的"));+ }++ /// 没有文字的装饰性 block 该消失,否则正文里全是空行。+ #[test]+ fn a_divider_renders_to_nothing() {+ let b = serde_json::json!({ "type": "divider", "divider": {} });+ assert!(render_block(&b).is_none());+ }++ /// 文件名不能带路径分隔符或换行。+ #[test]+ fn a_slug_is_safe_as_a_filename() {+ assert_eq!(slug("2026 Q3 / 复盘"), "2026-Q3---复盘");+ assert_eq!(slug("///"), "untitled");+ assert!(slug(&"x".repeat(200)).chars().count() <= 60);+ }++ /// 真连一个 Notion workspace。**没有模拟器**——Notion 是闭源 SaaS,+ /// 开源的替代品(AppFlowy、AFFiNE)不说这套 API。所以这条只有在+ /// 有人给出真 token 时才跑,CI 上永远跳过。+ ///+ /// ```text+ /// # 设置 → 我的连接 → 新建内部集成,然后把一个页面分享给它+ /// UTOPIA_NOTION_TEST_TOKEN=ntn_xxx cargo test -p utopia-server notion+ /// ```+ #[tokio::test]+ async fn it_reads_from_a_real_workspace() -> anyhow::Result<()> {+ let Ok(token) = std::env::var("UTOPIA_NOTION_TEST_TOKEN") else {+ eprintln!("跳过:未设 UTOPIA_NOTION_TEST_TOKEN");+ return Ok(());+ };+ let (pages, _) = fetch(&token, None).await?;+ assert!(+ !pages.is_empty(),+ "一页都没有——integration 可能没有被分享任何页面"+ );+ let p = &pages[0];+ assert!(+ p.external_key.starts_with("notion://"),+ "{}",+ p.external_key+ );+ assert!(p.text.starts_with("# "), "正文该以标题开头");+ assert!(+ p.last_edited.is_some(),+ "last_edited_time 是 doc_time 的来源"+ );+ Ok(())+ }+}diff --git a/web/src/api.ts b/web/src/api.tsindex bf46b22a..59302ca0 100644--- a/web/src/api.ts+++ b/web/src/api.ts@@ -189,6 +189,7 @@ export interface SourceView {| "azure_blob"| "gcs"| "webdav"+ | "notion"| "memory"| "upload";name: string;diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.tsindex c21e6bcb..09bc4899 100644--- a/web/src/i18n/en.ts+++ b/web/src/i18n/en.ts@@ -463,6 +463,7 @@ export const en = {azure_blob: "Azure Blob",gcs: "Google Cloud Storage",webdav: "WebDAV",+ notion: "Notion",},sourceKindHints: {folder:@@ -491,6 +492,10 @@ export const en = {"Reads documents off a WebDAV share — Nextcloud, ownCloud, Synology, or anything " +"else speaking the protocol. Walks folders one level at a time; each file becomes " +"a document dated by its last modification.",+ notion:+ "Syncs the pages a Notion integration can see — share a page with the integration " ++ "and it appears here. Dated by when the page was last edited, which is the page's " ++ "own clock rather than ours.",api: "External systems push JSON documents here, authenticated with this source's own token.",custom:"Polls a URL you control on a schedule — your service returns JSON items and Utopia keeps them in sync.",@@ -550,6 +555,8 @@ export const en = {davPathField: "Folder (optional — defaults to the root)",davUserField: "Username",davPassField: "Password",+ notionTokenField: "Internal integration token",+ notionQueryField: "Search term (optional — empty takes every shared page)",tokenField: "GitHub token (optional, never shown again)",includePullRequests: "Treat pull requests as tickets too",interval: "Sync schedule",diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.tsindex 8cd9bb05..ab194530 100644--- a/web/src/i18n/zh.ts+++ b/web/src/i18n/zh.ts@@ -429,6 +429,7 @@ export const zh: Strings = {azure_blob: "Azure Blob",gcs: "Google Cloud Storage",webdav: "WebDAV",+ notion: "Notion",},sourceKindHints: {folder:@@ -455,6 +456,9 @@ export const zh: Strings = {webdav:"从 WebDAV 网盘读文档——Nextcloud、ownCloud、群晖,以及一切说这套协议的。" +"逐层走目录,每个文件成为一篇文档,日期取它最后一次修改的时刻。",+ notion:+ "同步一个 Notion 集成能看见的页面——把页面分享给集成,它就会出现在这里。" ++ "日期取页面最后一次编辑的时刻,那是页面自己的时钟,不是我们抓它的时刻。",api: "外部系统把 JSON 文档推送到这里,用这个来源自己的令牌认证。",custom:"按计划轮询一个你控制的 URL——你的服务返回 JSON 条目,Utopia 保持同步。",@@ -513,6 +517,8 @@ export const zh: Strings = {davPathField: "目录(可选——默认从根开始)",davUserField: "用户名",davPassField: "密码",+ notionTokenField: "内部集成令牌",+ notionQueryField: "搜索词(可选——留空则取全部已分享页面)",tokenField: "GitHub 令牌(可选,不再显示)",includePullRequests: "把 PR 也当工单收进来",interval: "同步计划",diff --git a/web/src/pages/Library.tsx b/web/src/pages/Library.tsxindex 3e2cbc35..1d732a28 100644--- a/web/src/pages/Library.tsx+++ b/web/src/pages/Library.tsx@@ -1274,6 +1274,7 @@ function SourceModal({| "azure_blob"| "gcs"| "webdav"+ | "notion">("folder");const [name, setName] = useState("");const [icon, setIcon] = useState<string | null>(null);@@ -1297,6 +1298,8 @@ function SourceModal({const [davPath, setDavPath] = useState("");const [davUser, setDavUser] = useState("");const [davPass, setDavPass] = useState("");+ const [notionToken, setNotionToken] = useState("");+ const [notionQuery, setNotionQuery] = useState("");// PR 在 GitHub 的模型里也是工单。默认不收——问「工单」要的是工单;// 但有些仓库的决策记录实际写在 PR 描述里,所以给个开关const [includePrs, setIncludePrs] = useState(false);@@ -1314,7 +1317,8 @@ function SourceModal({kind === "s3" ||kind === "azure_blob" ||kind === "gcs" ||- kind === "webdav";+ kind === "webdav" ||+ kind === "notion";const create = useMutation({mutationFn: () => {@@ -1374,7 +1378,14 @@ function SourceModal({...(davUser.trim() ? { username: davUser.trim() } : {}),...(davPass ? { password: davPass } : {}),}- : {};+ : kind === "notion"+ ? {+ token: notionToken.trim(),+ ...(notionQuery.trim()+ ? { query: notionQuery.trim() }+ : {}),+ }+ : {};return api.createSource(kbId, {kind,name: name.trim(),@@ -1396,8 +1407,10 @@ function SourceModal({? feedUrl.trim(): kind === "custom"? endpoint.trim()- : kind === "webdav"- ? davUrl.trim()+ : kind === "notion"+ ? notionToken.trim()+ : kind === "webdav"+ ? davUrl.trim(): kind === "s3" || kind === "azure_blob" || kind === "gcs"? s3Bucket.trim(): true);@@ -1440,6 +1453,7 @@ function SourceModal({"azure_blob","gcs","webdav",+ "notion","api","custom",] as const@@ -1714,6 +1728,28 @@ function SourceModal({)}</>)}+ {kind === "notion" && (+ <>+ {field(+ S.library.notionTokenField,+ <input+ className="input-dark w-full px-3 py-2 text-sm font-mono"+ type="password"+ placeholder="ntn_..."+ value={notionToken}+ onChange={(e) => setNotionToken(e.target.value)}+ />,+ )}+ {field(+ S.library.notionQueryField,+ <input+ className="input-dark w-full px-3 py-2 text-sm font-mono"+ value={notionQuery}+ onChange={(e) => setNotionQuery(e.target.value)}+ />,+ )}+ </>+ )}{kind === "github_issues" && (<>{field(diff --git a/web/src/pages/SourcesRail.tsx b/web/src/pages/SourcesRail.tsxindex 64f250fe..25de4d28 100644--- a/web/src/pages/SourcesRail.tsx+++ b/web/src/pages/SourcesRail.tsx@@ -71,6 +71,7 @@ export const KIND_ICON = {azure_blob: Cloud,gcs: Cloud,webdav: FolderOpen,+ notion: Notebook,memory: Brain,upload: Upload,} as const;@@ -86,6 +87,7 @@ export const SYNCING_KINDS = new Set(["azure_blob","gcs","webdav",+ "notion",]);export const SYNC_DOT: Record<SourceView["last_sync_status"], string> = {