Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(lexical): convert Notion pages to Lexical in lexical-service (#8591)
Wolf Mermelstein47c13ff42200
diff --git a/Cargo.lock b/Cargo.lockindex 90bc8224ea..916778ec0e 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -11267,6 +11267,7 @@ version = "0.1.0"dependencies = ["agent_fold","anyhow",+ "base64 0.22.1","macro_tower_layers","messages","reqwest 0.13.4",diff --git a/crates/documents/AGENTS.md b/crates/documents/AGENTS.mdindex d599ceafbb..3a59ee95c4 100644--- a/crates/documents/AGENTS.md+++ b/crates/documents/AGENTS.md@@ -54,6 +54,10 @@ word docs, Illustrator/Photoshop files).- Add a document route: `src/domain/service.rs` (new domain method) →`src/inbound/axum_router.rs` → `src/domain/ports.rs` if it needs a newoutbound capability.+- Create a document from a Notion page: `DocumentCreator::create_notion_markdown`+ (`src/domain/create.rs`) with a `NotionMarkdown` (`src/domain/notion_markdown.rs`);+ lexical-service converts it through `MarkdownInitializationPort::initialize_existing_notion`+ and returns the Macro markdown that mention tracking reads.- Change content storage/markdown lifecycle: `src/domain/content.rs` →`src/outbound/s3_markdown_source.rs` / `src/outbound/markdown_init.rs` →`src/bin/backfill_markdown_content_location.rs` if existing documents needdiff --git a/crates/documents/src/bin/backfill_markdown_content_location.rs b/crates/documents/src/bin/backfill_markdown_content_location.rsindex afc4bc4548..fd1e8bfe98 100644--- a/crates/documents/src/bin/backfill_markdown_content_location.rs+++ b/crates/documents/src/bin/backfill_markdown_content_location.rs@@ -237,6 +237,14 @@ impl MarkdownInitializationPort for OptionalMarkdownInitializer {}}}++ async fn initialize_existing_notion(+ &self,+ _document_id: &str,+ _notion: &documents::domain::notion_markdown::NotionMarkdown,+ ) -> Result<documents::domain::ports::markdown::MarkdownSnapshot, DocumentError> {+ unimplemented!("the markdown backfill never imports Notion pages")+ }}fn print_summary(report: &MarkdownBackfillReport, apply: bool) {diff --git a/crates/documents/src/domain.rs b/crates/documents/src/domain.rsindex 317d04eedc..ad3023f3c7 100644--- a/crates/documents/src/domain.rs+++ b/crates/documents/src/domain.rs@@ -31,6 +31,8 @@ pub mod upload_finalize;pub mod legacy_office_upgrade;pub mod models;+/// Notion page bodies a document can be created from.+pub mod notion_markdown;#[cfg(feature = "axum")]pub mod permission_token;pub mod response;diff --git a/crates/documents/src/domain/create.rs b/crates/documents/src/domain/create.rsindex 2bb451a0b8..eb438f6875 100644--- a/crates/documents/src/domain/create.rs+++ b/crates/documents/src/domain/create.rs@@ -22,10 +22,11 @@ use crate::domain::content::{DocumentContent, DocumentContentLocation};use crate::domain::models::{CreateTaskRequest, DocumentError, EMPTY_SHA256, InitialLinkShare, NewDocument, PropertyInput,};+use crate::domain::notion_markdown::NotionMarkdown;use crate::domain::ports::create::{DocumentBytesUpload, DocumentBytesUploadPort, DocumentCreationService,};-use crate::domain::ports::markdown::MarkdownInitializationPort;+use crate::domain::ports::markdown::{MarkdownInitializationPort, MarkdownSnapshot};use crate::domain::ports::mentions::DocumentMentionTrackingPort;use crate::domain::response::CreateDocumentResponseData;@@ -221,6 +222,25 @@ impl NewMarkdownTextDocument {}}+/// A backend-created markdown document whose content comes from a Notion+/// page: lexical-service converts the page, resolving its references+/// through the links it carries.+#[derive(Debug, Clone)]+pub struct NewNotionDocument {+ /// Common document metadata.+ pub metadata: NewDocumentMetadata,+ /// The Notion page body and its links.+ pub notion: NotionMarkdown,+ /// Markdown subtype.+ pub subtype: MarkdownSubtype,+}++/// Where a new markdown document's content comes from.+enum MarkdownContent {+ Markdown(String),+ Notion(NotionMarkdown),+}+/// A backend-created plaintext document whose file type determines the/// creation lifecycle.#[derive(Debug, Clone)]@@ -583,6 +603,44 @@ wheremarkdown,subtype,} = document;+ self.create_markdown(+ principal,+ metadata,+ subtype,+ MarkdownContent::Markdown(markdown),+ )+ .await+ }++ /// Create a markdown document from a Notion page and initialize+ /// sync-service.+ #[tracing::instrument(skip(self, document), err)]+ pub async fn create_notion_markdown(+ &self,+ principal: &CreationPrincipal,+ document: NewNotionDocument,+ ) -> Result<CreatedDocument, DocumentError> {+ let NewNotionDocument {+ metadata,+ notion,+ subtype,+ } = document;+ self.create_markdown(+ principal,+ metadata,+ subtype,+ MarkdownContent::Notion(notion),+ )+ .await+ }++ async fn create_markdown(+ &self,+ principal: &CreationPrincipal,+ metadata: NewDocumentMetadata,+ subtype: MarkdownSubtype,+ content: MarkdownContent,+ ) -> Result<CreatedDocument, DocumentError> {let task = match &subtype {MarkdownSubtype::Note| MarkdownSubtype::Snippet@@ -632,10 +690,20 @@ where.clone();let finalize_result = async {- let initial_snapshot = self- .markdown_initializer- .initialize_existing_markdown(&document_id, &markdown)- .await?;+ let initialized = match &content {+ MarkdownContent::Markdown(markdown) => MarkdownSnapshot {+ snapshot: self+ .markdown_initializer+ .initialize_existing_markdown(&document_id, markdown)+ .await?,+ markdown: markdown.clone(),+ },+ MarkdownContent::Notion(notion) => {+ self.markdown_initializer+ .initialize_existing_notion(&document_id, notion)+ .await?+ }+ };self.document_service.set_document_content(@@ -663,12 +731,15 @@ where.await?;}- Ok(initial_snapshot)+ Ok(initialized)}.await;- let initial_snapshot = match finalize_result {- Ok(initial_snapshot) => initial_snapshot,+ let MarkdownSnapshot {+ snapshot: initial_snapshot,+ markdown,+ } = match finalize_result {+ Ok(initialized) => initialized,Err(error) => {self.cleanup_created_document(&document_id).await;return Err(error);diff --git a/crates/documents/src/domain/create/test.rs b/crates/documents/src/domain/create/test.rsindex d787a99b0b..8c4f1508cb 100644--- a/crates/documents/src/domain/create/test.rs+++ b/crates/documents/src/domain/create/test.rs@@ -7,14 +7,15 @@ use model_owner::CreationPrincipal;use super::{DocumentCreator, MarkdownSubtype, NewDocumentMetadata, NewMarkdownTextDocument,- NewPlainTextDocument, RepoDocumentKind, RepoDocumentSubtype, file_shas,+ NewNotionDocument, NewPlainTextDocument, RepoDocumentKind, RepoDocumentSubtype, file_shas,};use crate::domain::content::DocumentContent;use crate::domain::models::{CreateTaskRequest, DocumentError, InitialLinkShare, NewDocument};+use crate::domain::notion_markdown::NotionMarkdown;use crate::domain::ports::create::{DocumentBytesUpload, DocumentBytesUploadPort, DocumentCreationService,};-use crate::domain::ports::markdown::MarkdownInitializationPort;+use crate::domain::ports::markdown::{MarkdownInitializationPort, MarkdownSnapshot};use crate::domain::ports::mentions::DocumentMentionTrackingPort;use crate::domain::response::{CreateDocumentResponseData, DocumentResponse, DocumentResponseMetadataWithContent,@@ -128,6 +129,17 @@ impl MarkdownInitializationPort for FakeMarkdownInitializer {) -> Result<Vec<u8>, DocumentError> {Ok(vec![1, 2, 3])}++ async fn initialize_existing_notion(+ &self,+ _document_id: &str,+ notion: &NotionMarkdown,+ ) -> Result<MarkdownSnapshot, DocumentError> {+ Ok(MarkdownSnapshot {+ snapshot: vec![4, 5, 6],+ markdown: format!("converted: {}", notion.markdown),+ })+ }}struct RecordingMarkdownInitializer {@@ -149,6 +161,14 @@ impl MarkdownInitializationPort for RecordingMarkdownInitializer {}Ok(vec![1, 2, 3])}++ async fn initialize_existing_notion(+ &self,+ _document_id: &str,+ _notion: &NotionMarkdown,+ ) -> Result<MarkdownSnapshot, DocumentError> {+ panic!("unexpected initialize_existing_notion call")+ }}struct FakeBytesUploader;@@ -324,6 +344,38 @@ async fn markdown_creation_tracks_the_seeded_mentions_once() {);}+#[tokio::test]+async fn notion_creation_tracks_the_mentions_of_the_converted_markdown() {+ let tracker = RecordingMentionTracker::default();++ let created = creator(&tracker)+ .create_notion_markdown(+ &CreationPrincipal::User(owner()),+ NewNotionDocument {+ metadata: NewDocumentMetadata::new("Roadmap"),+ notion: NotionMarkdown {+ markdown: "<mention-page url=\"https://www.notion.so/p\">Plan</mention-page>"+ .to_string(),+ ..NotionMarkdown::default()+ },+ subtype: MarkdownSubtype::Note,+ },+ )+ .await+ .expect("creation should succeed");++ assert_eq!(created.initial_snapshot(), Some([4, 5, 6].as_slice()));+ assert_eq!(+ tracker.calls(),+ vec![(+ DOCUMENT_ID.to_string(),+ Some("macro|[email protected]".to_string()),+ "converted: <mention-page url=\"https://www.notion.so/p\">Plan</mention-page>"+ .to_string(),+ )]+ );+}+#[tokio::test]async fn mention_tracking_failure_does_not_fail_creation() {let tracker = RecordingMentionTracker::failing();diff --git a/crates/documents/src/domain/notion_markdown.rs b/crates/documents/src/domain/notion_markdown.rsnew file mode 100644index 0000000000..8f81c0adf3--- /dev/null+++ b/crates/documents/src/domain/notion_markdown.rs@@ -0,0 +1,34 @@+//! A Notion page body, in Notion's enhanced markdown, with what its page and+//! people references resolve to in Macro. Converted to document content by+//! lexical-service rather than parsed here.++use std::collections::HashMap;++/// A Notion page body and where its references point.+#[derive(Debug, Clone, Default, PartialEq, Eq)]+pub struct NotionMarkdown {+ /// The page in Notion's enhanced markdown.+ pub markdown: String,+ /// Notion page id (dashless, lowercase) → the Macro document it became.+ pub pages: HashMap<String, NotionPageLink>,+ /// Notion user id (dashless, lowercase) → the Macro user they are.+ pub users: HashMap<String, NotionUserLink>,+}++/// The Macro document a Notion page became.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct NotionPageLink {+ /// The document id.+ pub document_id: uuid::Uuid,+ /// Its name.+ pub name: String,+}++/// The Macro user a Notion person is.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct NotionUserLink {+ /// The Macro user id.+ pub user_id: String,+ /// Their email.+ pub email: String,+}diff --git a/crates/documents/src/domain/ports/markdown.rs b/crates/documents/src/domain/ports/markdown.rsindex 1f5da33e80..7ab5e038a2 100644--- a/crates/documents/src/domain/ports/markdown.rs+++ b/crates/documents/src/domain/ports/markdown.rs@@ -3,6 +3,16 @@use std::future::Future;use crate::domain::models::DocumentError;+use crate::domain::notion_markdown::NotionMarkdown;++/// A new document's Loro snapshot and its content as Macro markdown.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct MarkdownSnapshot {+ /// The Loro snapshot.+ pub snapshot: Vec<u8>,+ /// The same content as Macro markdown.+ pub markdown: String,+}/// Initializes markdown document content in the collaborative editor backend.pub trait MarkdownInitializationPort: Send + Sync {@@ -12,6 +22,14 @@ pub trait MarkdownInitializationPort: Send + Sync {document_id: &str,markdown: &str,) -> impl Future<Output = Result<Vec<u8>, DocumentError>> + Send;++ /// Initialize an already-created markdown document from a Notion page+ /// and return its snapshot with the Macro markdown it holds.+ fn initialize_existing_notion(+ &self,+ document_id: &str,+ notion: &NotionMarkdown,+ ) -> impl Future<Output = Result<MarkdownSnapshot, DocumentError>> + Send;}/// Utilities from the lexical service@@ -22,6 +40,12 @@ pub trait LexicalSnapshotPort: Send + Sync + 'static {&self,markdown: &str,) -> impl Future<Output = anyhow::Result<Vec<u8>>> + Send;++ /// Convert a Notion page to loro via lexical using lexical service+ fn notion_to_loro_snapshot(+ &self,+ notion: &NotionMarkdown,+ ) -> impl Future<Output = anyhow::Result<MarkdownSnapshot>> + Send;}/// Utilities for the worker/dodiff --git a/crates/documents/src/domain/service/tests.rs b/crates/documents/src/domain/service/tests.rsindex de226809c5..7ac863dd4f 100644--- a/crates/documents/src/domain/service/tests.rs+++ b/crates/documents/src/domain/service/tests.rs@@ -2327,6 +2327,14 @@ impl crate::domain::ports::markdown::MarkdownInitializationPort for RejectUnexpe) -> Result<Vec<u8>, DocumentError> {panic!("failed repository creation must not initialize markdown");}++ async fn initialize_existing_notion(+ &self,+ _: &str,+ _: &crate::domain::notion_markdown::NotionMarkdown,+ ) -> Result<crate::domain::ports::markdown::MarkdownSnapshot, DocumentError> {+ panic!("failed repository creation must not initialize markdown");+ }}impl crate::domain::ports::create::DocumentBytesUploadPort for RejectUnexpectedFinalization {diff --git a/crates/documents/src/outbound/markdown_init.rs b/crates/documents/src/outbound/markdown_init.rsindex 3bd1af0b1e..2592d3d85e 100644--- a/crates/documents/src/outbound/markdown_init.rs+++ b/crates/documents/src/outbound/markdown_init.rs@@ -9,12 +9,16 @@ use std::sync::Arc;use std::time::Duration;use lexical_client::LexicalClient;+use lexical_client::notion::{+ NotionSnapshotLinks, NotionSnapshotPage, NotionSnapshotRequest, NotionSnapshotUser,+};use sync_service_client::{SyncServiceClient, initialize::SnapshotAlreadyExists};use tokio_retry::{RetryIf, strategy::FixedInterval};use crate::domain::models::DocumentError;+use crate::domain::notion_markdown::NotionMarkdown;use crate::domain::ports::markdown::{- LexicalSnapshotPort, MarkdownInitializationPort, SyncInitializeSnapshotPort,+ LexicalSnapshotPort, MarkdownInitializationPort, MarkdownSnapshot, SyncInitializeSnapshotPort,};impl LexicalSnapshotPort for LexicalClient {@@ -24,6 +28,44 @@ impl LexicalSnapshotPort for LexicalClient {) -> impl Future<Output = anyhow::Result<Vec<u8>>> + Send {LexicalClient::markdown_to_loro_snapshot(self, markdown)}++ async fn notion_to_loro_snapshot(+ &self,+ notion: &NotionMarkdown,+ ) -> anyhow::Result<MarkdownSnapshot> {+ let request = NotionSnapshotRequest {+ markdown: ¬ion.markdown,+ links: NotionSnapshotLinks {+ pages: notion+ .pages+ .iter()+ .map(|(notion_id, page)| {+ let page = NotionSnapshotPage {+ document_id: page.document_id.to_string(),+ name: page.name.clone(),+ };+ (notion_id.clone(), page)+ })+ .collect(),+ users: notion+ .users+ .iter()+ .map(|(notion_id, user)| {+ let user = NotionSnapshotUser {+ user_id: user.user_id.clone(),+ email: user.email.clone(),+ };+ (notion_id.clone(), user)+ })+ .collect(),+ },+ };+ let converted = LexicalClient::notion_to_loro_snapshot(self, &request).await?;+ Ok(MarkdownSnapshot {+ snapshot: converted.snapshot,+ markdown: converted.markdown,+ })+ }}impl SyncInitializeSnapshotPort for SyncServiceClient {@@ -99,16 +141,47 @@ where.map_err(DocumentError::Internal)?};+ self.initialize_sync(document_id, &loro_snapshot).await?;+ Ok(loro_snapshot)+ }++ #[tracing::instrument(skip(self, notion), fields(markdown_bytes = notion.markdown.len()), err)]+ async fn initialize_existing_notion(+ &self,+ document_id: &str,+ notion: &NotionMarkdown,+ ) -> Result<MarkdownSnapshot, DocumentError> {+ let converted = self+ .lexical_client+ .notion_to_loro_snapshot(notion)+ .await+ .map_err(DocumentError::Internal)?;+ self.initialize_sync(document_id, &converted.snapshot)+ .await?;+ Ok(converted)+ }+}++impl<L, S> LexicalSyncMarkdownInitializer<L, S>+where+ S: SyncInitializeSnapshotPort + 'static,+{+ /// Hand `loro_snapshot` to sync-service, awaited or detached.+ async fn initialize_sync(+ &self,+ document_id: &str,+ loro_snapshot: &[u8],+ ) -> Result<(), DocumentError> {match self.sync_initialization {SyncInitialization::Awaited => {- initialize_with_retry(&*self.sync_service_client, document_id, &loro_snapshot)+ initialize_with_retry(&*self.sync_service_client, document_id, loro_snapshot).await.map_err(DocumentError::Internal)?;}SyncInitialization::Detached => {let sync_service_client = Arc::clone(&self.sync_service_client);let document_id = document_id.to_owned();- let snapshot = loro_snapshot.clone();+ let snapshot = loro_snapshot.to_vec();tokio::spawn(async move {if let Err(error) =initialize_with_retry(&*sync_service_client, &document_id, &snapshot).await@@ -119,7 +192,7 @@ where}}- Ok(loro_snapshot)+ Ok(())}}diff --git a/crates/lexical_client/AGENTS.md b/crates/lexical_client/AGENTS.mdindex 39858ec4c9..0b5a4b831e 100644--- a/crates/lexical_client/AGENTS.md+++ b/crates/lexical_client/AGENTS.md@@ -13,6 +13,9 @@ to read or render document content.- `src/parse_markdown.rs` (+ `src/parse_markdown/test.rs`): markdown → Lexicalnode tree parsing.- `src/types.rs`: the Lexical node/document types shared with the service.+- `src/notion.rs`: `notion_to_loro_snapshot` (`POST /snapshot/notion`) and its+ hand-copied request types; keep them in step with+ `services/lexical-service/src/endpoints/notion-snapshot.ts`.## Scopediff --git a/crates/lexical_client/Cargo.toml b/crates/lexical_client/Cargo.tomlindex 3bb07ebd16..969f9909e1 100644--- a/crates/lexical_client/Cargo.toml+++ b/crates/lexical_client/Cargo.toml@@ -7,6 +7,7 @@ version = "0.1.0"[dependencies]agent_fold = { path = "../folds/agent_fold", default-features = false }anyhow = { workspace = true }+base64 = { workspace = true }macro_tower_layers = { path = "../macro_tower_layers" }messages = { path = "../messages" }trigger_context = { path = "../trigger_context" }diff --git a/crates/lexical_client/src/lib.rs b/crates/lexical_client/src/lib.rsindex 4d00456c90..e9cfea21ba 100644--- a/crates/lexical_client/src/lib.rs+++ b/crates/lexical_client/src/lib.rs@@ -1,4 +1,5 @@pub(crate) static INTERNAL_ACCESS_HEADER: &str = "x-internal-auth-key";+pub mod notion;pub mod parse_markdown;pub mod types;diff --git a/crates/lexical_client/src/notion.rs b/crates/lexical_client/src/notion.rsnew file mode 100644index 0000000000..e4514a1684--- /dev/null+++ b/crates/lexical_client/src/notion.rs@@ -0,0 +1,150 @@+//! Converting a Notion page's enhanced markdown into a new document's Loro+//! snapshot (`POST /snapshot/notion`).++use std::collections::HashMap;++use anyhow::{Context, Result};+use base64::Engine;++use super::LexicalClient;+use crate::parse_markdown::check_response;++/// A Notion page body and what its references resolve to.+#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize)]+pub struct NotionSnapshotRequest<'a> {+ /// The page in Notion's enhanced markdown.+ pub markdown: &'a str,+ /// Where its page and people references point in Macro.+ pub links: NotionSnapshotLinks,+}++/// Notion ids (dashless, lowercase) mapped to the Macro objects they became.+#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize)]+pub struct NotionSnapshotLinks {+ /// Notion page id → the Macro document it became.+ pub pages: HashMap<String, NotionSnapshotPage>,+ /// Notion user id → the Macro user they are.+ pub users: HashMap<String, NotionSnapshotUser>,+}++/// The Macro document a Notion page became.+#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]+#[serde(rename_all = "camelCase")]+pub struct NotionSnapshotPage {+ /// The document id.+ pub document_id: String,+ /// Its name.+ pub name: String,+}++/// The Macro user a Notion person is.+#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]+#[serde(rename_all = "camelCase")]+pub struct NotionSnapshotUser {+ /// The Macro user id.+ pub user_id: String,+ /// Their email.+ pub email: String,+}++/// A converted Notion page.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct NotionSnapshot {+ /// The new document's Loro snapshot.+ pub snapshot: Vec<u8>,+ /// The same content as Macro markdown.+ pub markdown: String,+}++#[derive(serde::Deserialize)]+struct NotionSnapshotResponse {+ snapshot: String,+ markdown: String,+}++impl LexicalClient {+ /// Convert a Notion page to a new document's Loro snapshot.+ #[tracing::instrument(+ skip(self, request),+ fields(+ markdown_bytes = request.markdown.len(),+ pages = request.links.pages.len(),+ users = request.links.users.len()+ ),+ err+ )]+ pub async fn notion_to_loro_snapshot(+ &self,+ request: &NotionSnapshotRequest<'_>,+ ) -> Result<NotionSnapshot> {+ let mut headers = reqwest::header::HeaderMap::new();+ macro_tower_layers::inject_trace_headers(&mut headers);+ let response = check_response(+ self.client+ .post(format!("{}/snapshot/notion", self.url))+ .headers(headers)+ .json(request)+ .send()+ .await?,+ )+ .await?;+ let body: NotionSnapshotResponse = response.json().await.context("unexpected response")?;+ Ok(NotionSnapshot {+ snapshot: base64::engine::general_purpose::STANDARD+ .decode(body.snapshot)+ .context("snapshot is not base64")?,+ markdown: body.markdown,+ })+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn the_request_matches_the_lexical_service_schema() {+ let request = NotionSnapshotRequest {+ markdown: "See <mention-page url=\"https://www.notion.so/p\">Plan</mention-page>",+ links: NotionSnapshotLinks {+ pages: [(+ "3fb607323e2e80629886fc0c028d50a7".to_string(),+ NotionSnapshotPage {+ document_id: "0e5b6a2c-7c39-4a63-9a0c-5c1f2f6c0001".to_string(),+ name: "Plan".to_string(),+ },+ )]+ .into(),+ users: [(+ "12345678123412341234123456789abc".to_string(),+ NotionSnapshotUser {+ user_id: "macro|[email protected]".to_string(),+ email: "[email protected]".to_string(),+ },+ )]+ .into(),+ },+ };++ assert_eq!(+ serde_json::to_value(&request).unwrap(),+ serde_json::json!({+ "markdown": "See <mention-page url=\"https://www.notion.so/p\">Plan</mention-page>",+ "links": {+ "pages": {+ "3fb607323e2e80629886fc0c028d50a7": {+ "documentId": "0e5b6a2c-7c39-4a63-9a0c-5c1f2f6c0001",+ "name": "Plan"+ }+ },+ "users": {+ "12345678123412341234123456789abc": {+ "userId": "macro|[email protected]",+ "email": "[email protected]"+ }+ }+ }+ })+ );+ }+}diff --git a/crates/lexical_client/src/parse_markdown.rs b/crates/lexical_client/src/parse_markdown.rsindex 0632fb3caf..b3f05ea362 100644--- a/crates/lexical_client/src/parse_markdown.rs+++ b/crates/lexical_client/src/parse_markdown.rs@@ -451,7 +451,7 @@ impl From<LexicalResponseItem> for MarkdownParseResult {}}-async fn check_response(response: reqwest::Response) -> Result<reqwest::Response> {+pub(crate) async fn check_response(response: reqwest::Response) -> Result<reqwest::Response> {if response.status() == reqwest::StatusCode::OK {return Ok(response);}diff --git a/packages/lexical-core/markdown-loro-snapshot.ts b/packages/lexical-core/markdown-loro-snapshot.tsindex 1d89103e7f..490f87b4b3 100644--- a/packages/lexical-core/markdown-loro-snapshot.ts+++ b/packages/lexical-core/markdown-loro-snapshot.ts@@ -50,12 +50,20 @@ export function markdownToSerializedEditorState() as SerializedEditorState;}+/** A new document's snapshot holding `state`, seeded from the golden base. */+export function serializedEditorStateToLoroSnapshot(+ state: SerializedEditorState+): Promise<Uint8Array | undefined> {+ return rawMarkdownStateToLoroSnapshot(state as any, MARKDOWN_GOLDEN);+}+export async function markdownToLoroSnapshot(markdown: string): Promise<Uint8Array | undefined> {// Blank markdown is exactly the golden — return it verbatim so all empty docs// share identical bytes and skip the mirror round-trip.if (markdown === '') return MARKDOWN_GOLDEN;- const state = markdownToSerializedEditorState(markdown);- return rawMarkdownStateToLoroSnapshot(state as any, MARKDOWN_GOLDEN);+ return serializedEditorStateToLoroSnapshot(+ markdownToSerializedEditorState(markdown)+ );}diff --git a/packages/lexical-core/notion-markdown.ts b/packages/lexical-core/notion-markdown.tsnew file mode 100644index 0000000000..4b5ee59d61--- /dev/null+++ b/packages/lexical-core/notion-markdown.ts@@ -0,0 +1,78 @@+import { createHeadlessEditor } from '@lexical/headless';+import { $convertFromMarkdownString } from '@lexical/markdown';+import type { SerializedEditorState } from 'lexical';+import { MARKDOWN_GOLDEN } from './markdown-golden.1';+import { serializedEditorStateToLoroSnapshot } from './markdown-loro-snapshot';+import { NodeReplacements, SupportedNodeTypes } from './node-list';+import type { NotionLinks } from './notion/links';+import { notionLines, notionTransformers } from './notion/transformers';+import { $updateAllNodeIds } from './plugins/nodeIdPlugin';+import { serializedEditorStateToMarkdown } from './utils/markdown-state';++export type {+ NotionLinks,+ NotionPageLink,+ NotionUserLink,+} from './notion/links';++/**+ * Converts Notion enhanced markdown to a serialized Lexical editor state with+ * durable node ids. Page and people references resolve through `links`.+ */+export function notionMarkdownToSerializedEditorState(+ markdown: string,+ links: NotionLinks+): SerializedEditorState {+ const editor = createHeadlessEditor({+ nodes: [...SupportedNodeTypes, ...NodeReplacements],+ onError: (error) => {+ throw error;+ },+ });+ editor.update(+ () => {+ $convertFromMarkdownString(+ notionLines(markdown),+ notionTransformers(links),+ undefined,+ true+ );+ },+ { discrete: true }+ );+ editor.update(+ () => {+ $updateAllNodeIds({+ idToNodeKeyMap: new Map(),+ nodeKeyToIdMap: new Map(),+ });+ },+ { discrete: true }+ );+ return editor.getEditorState().toJSON();+}++/** A new document's snapshot, and the same content as Macro markdown. */+export type NotionSnapshot = {+ snapshot: Uint8Array;+ markdown: string;+};++/**+ * Converts Notion enhanced markdown to a Loro snapshot for a new document,+ * along with that document's Macro markdown (for mention tracking).+ */+export async function notionMarkdownToLoroSnapshot(+ markdown: string,+ links: NotionLinks+): Promise<NotionSnapshot> {+ if (markdown.trim() === '') {+ return { snapshot: MARKDOWN_GOLDEN, markdown: '' };+ }+ const state = notionMarkdownToSerializedEditorState(markdown, links);+ const snapshot = await serializedEditorStateToLoroSnapshot(state);+ if (!snapshot) {+ throw new Error('Failed to export the Notion page as a Loro snapshot');+ }+ return { snapshot, markdown: serializedEditorStateToMarkdown(state) };+}diff --git a/packages/lexical-core/notion/links.ts b/packages/lexical-core/notion/links.tsnew file mode 100644index 0000000000..245bf6d6bb--- /dev/null+++ b/packages/lexical-core/notion/links.ts@@ -0,0 +1,54 @@+/**+ * What a Notion import already knows about the objects a page refers to:+ * the Macro documents its pages became and the Macro users its people are.+ * Keys are Notion ids in their dashless, lowercase form.+ */+export type NotionLinks = {+ pages: Record<string, NotionPageLink>;+ users: Record<string, NotionUserLink>;+};++/** The Macro document a Notion page became. */+export type NotionPageLink = {+ documentId: string;+ name: string;+};++/** The Macro user a Notion person is. */+export type NotionUserLink = {+ userId: string;+ email: string;+};++const NOTION_ID =+ /[0-9a-f]{8}-?[0-9a-f]{4}-?[0-9a-f]{4}-?[0-9a-f]{4}-?[0-9a-f]{12}/gi;++/**+ * Every Notion id in `reference` (a URL like+ * `https://www.notion.so/Plan-3fb6…#9a1c…`, `user://…`, or a bare id), last+ * first: a block anchor names the block itself, which is what an unknown or+ * child-page placeholder refers to.+ */+export function notionIdsIn(reference: string): string[] {+ return [...reference.matchAll(NOTION_ID)]+ .map((match) => match[0].replaceAll('-', '').toLowerCase())+ .reverse();+}++export function findNotionPage(+ links: NotionLinks,+ reference: string+): NotionPageLink | undefined {+ return notionIdsIn(reference)+ .map((id) => links.pages[id])+ .find((page) => page !== undefined);+}++export function findNotionUser(+ links: NotionLinks,+ reference: string+): NotionUserLink | undefined {+ return notionIdsIn(reference)+ .map((id) => links.users[id])+ .find((user) => user !== undefined);+}diff --git a/packages/lexical-core/notion/transformers.ts b/packages/lexical-core/notion/transformers.tsnew file mode 100644index 0000000000..b2a09d6bfa--- /dev/null+++ b/packages/lexical-core/notion/transformers.ts@@ -0,0 +1,638 @@+/**+ * Transformers that read Notion's enhanced markdown (the `GET+ * /v1/pages/{id}/markdown` and `notion-fetch` dialect) into Macro's Lexical+ * nodes. Run them ahead of {@link ALL_TRANSFORMERS}, which handle the+ * standard markdown in between.+ *+ * Every Notion construct becomes a real node where Macro has one (document,+ * user and date mentions, tables, check lists, equations, images) and+ * readable text where it has none (callouts become quotes, toggles and+ * columns flatten into their content). No Notion tag survives as raw text.+ *+ * Spec: https://developers.notion.com/guides/data-apis/enhanced-markdown+ */+import { $createLinkNode } from '@lexical/link';+import {+ $convertFromMarkdownString,+ type ElementTransformer,+ type MultilineElementTransformer,+ type TextMatchTransformer,+ type Transformer,+} from '@lexical/markdown';+import { $createQuoteNode, type QuoteNode } from '@lexical/rich-text';+import {+ $createTableCellNode,+ $createTableNode,+ $createTableRowNode,+ TableCellHeaderStates,+ TableCellNode,+ TableNode,+ TableRowNode,+} from '@lexical/table';+import {+ $createLineBreakNode,+ $createParagraphNode,+ $createTextNode,+ $isParagraphNode,+ type ElementNode,+ type LexicalNode,+ LineBreakNode,+ type TextNode,+} from 'lexical';+import { match } from 'ts-pattern';+import {+ $createDateMentionNode,+ DateMentionNode,+} from '../nodes/DateMentionNode';+import {+ $createDocumentMentionNode,+ DocumentMentionNode,+} from '../nodes/DocumentMentionNode';+import { $createImageNode, ImageNode } from '../nodes/ImageNode';+import {+ $createUserMentionNode,+ UserMentionNode,+} from '../nodes/UserMentionNode';+import { ALL_TRANSFORMERS } from '../transformers';+import { findNotionPage, findNotionUser, type NotionLinks } from './links';++type Attributes = Record<string, string>;++function attributes(raw: string | undefined): Attributes {+ const parsed: Attributes = {};+ for (const [, name, value] of (raw ?? '').matchAll(/([\w-]+)="([^"]*)"/g)) {+ parsed[name] = value+ .replaceAll('"', '"')+ .replaceAll('<', '<')+ .replaceAll('>', '>')+ .replaceAll('&', '&');+ }+ return parsed;+}++function isWebUrl(value: string | undefined): value is string {+ return value?.startsWith('https://') || value?.startsWith('http://') || false;+}++function withoutTags(value: string): string {+ return value.replace(/<[^>]+>/g, '').trim();+}++/** Replace `node` with a link to `url` labelled `label`, or with the label alone when `url` isn't a web link. */+function $replaceWithLink(+ node: TextNode,+ url: string | undefined,+ label: string+): TextNode {+ const text = $createTextNode(label);+ text.setFormat(node.getFormat());+ if (!isWebUrl(url)) {+ node.replace(text);+ return text;+ }+ const link = $createLinkNode(url);+ link.append(text);+ node.replace(link);+ return text;+}++/** A page or database reference: a mention of the Macro document it became, else a link back to Notion. */+function $replaceWithPageReference(+ node: TextNode,+ links: NotionLinks,+ attrs: Attributes,+ label: string+): TextNode | undefined {+ const reference = attrs.url ?? '';+ const page = findNotionPage(links, reference);+ if (page) {+ node.replace(+ $createDocumentMentionNode({+ documentId: page.documentId,+ documentName: page.name,+ blockName: 'md',+ })+ );+ return undefined;+ }+ const title = withoutTags(label) || attrs.title?.trim() || 'Notion page';+ return $replaceWithLink(node, reference, title);+}++/** Noon UTC for a bare date, so it lands on the same day in every time zone. */+function notionInstant(+ date: string | undefined,+ time: string | undefined,+ timeZone: string | undefined+): Date | undefined {+ if (!date || !/^\d{4}-\d{2}-\d{2}$/.test(date)) return undefined;+ if (!time || !/^\d{2}:\d{2}$/.test(time)) {+ return new Date(`${date}T12:00:00.000Z`);+ }+ const wallClock = new Date(`${date}T${time}:00.000Z`);+ if (!timeZone) return wallClock;+ try {+ const parts = new Intl.DateTimeFormat('en-US', {+ timeZone,+ hourCycle: 'h23',+ year: 'numeric',+ month: '2-digit',+ day: '2-digit',+ hour: '2-digit',+ minute: '2-digit',+ }).formatToParts(wallClock);+ const part = (type: Intl.DateTimeFormatPartTypes) =>+ Number(parts.find((candidate) => candidate.type === type)?.value);+ const zoned = Date.UTC(+ part('year'),+ part('month') - 1,+ part('day'),+ part('hour'),+ part('minute')+ );+ return new Date(wallClock.getTime() - (zoned - wallClock.getTime()));+ } catch {+ return wallClock;+ }+}++function dateLabel(date: string, time: string | undefined): string {+ const label = new Date(`${date}T12:00:00.000Z`).toLocaleDateString('en-US', {+ month: 'long',+ day: 'numeric',+ year: 'numeric',+ timeZone: 'UTC',+ });+ return time ? `${label} ${time}` : label;+}++function $dateMention(+ date: string | undefined,+ time: string | undefined,+ timeZone: string | undefined+): LexicalNode | undefined {+ const instant = notionInstant(date, time, timeZone);+ if (!instant || !date) return undefined;+ return $createDateMentionNode({+ date: instant.toISOString(),+ displayFormat: dateLabel(date, time),+ });+}++function $replaceWithDateMention(node: TextNode, attrs: Attributes): void {+ const start = $dateMention(attrs.start, attrs.startTime, attrs.timeZone);+ if (!start) {+ node.replace($createTextNode(attrs.start ?? ''));+ return;+ }+ node.replace(start);+ if (!attrs.end || attrs.end === attrs.start) return;+ const end = $dateMention(attrs.end, attrs.endTime, attrs.timeZone);+ if (!end) return;+ start.insertAfter(end);+ start.insertAfter($createTextNode(' – '));+}++function fileName(url: string | undefined): string | undefined {+ if (!isWebUrl(url)) return undefined;+ try {+ const last = new URL(url).pathname.split('/').pop();+ return last ? decodeURIComponent(last) : undefined;+ } catch {+ return undefined;+ }+}++function textMatch(+ regExp: RegExp,+ dependencies: TextMatchTransformer['dependencies'],+ replace: (node: TextNode, match: RegExpMatchArray) => TextNode | void+): TextMatchTransformer {+ return {+ dependencies,+ type: 'text-match',+ regExp,+ importRegExp: regExp,+ replace,+ };+}++function inlineTransformers(links: NotionLinks): TextMatchTransformer[] {+ return [+ textMatch(+ /<(mention-page|page|database|mention-database|mention-data-source)\b([^>]*)>(.*?)<\/\1>/,+ [DocumentMentionNode],+ (node, [, , attrs, label]) =>+ $replaceWithPageReference(node, links, attributes(attrs), label)+ ),+ textMatch(+ /<(?:mention-page|page|database|mention-database|mention-data-source)\b([^>]*?)\/>/,+ [DocumentMentionNode],+ (node, [, attrs]) =>+ $replaceWithPageReference(node, links, attributes(attrs), '')+ ),+ textMatch(+ /<mention-user\b([^>]*?)(?:\/>|>(.*?)<\/mention-user>)/,+ [UserMentionNode],+ (node, [, attrs, label]) => {+ const name = withoutTags(label ?? '');+ const user = findNotionUser(links, attributes(attrs).url ?? '');+ if (user) {+ node.replace(+ $createUserMentionNode({+ userId: user.userId,+ email: user.email,+ displayName: name || undefined,+ })+ );+ return;+ }+ if (!name) {+ node.remove();+ return;+ }+ node.setTextContent(`@${name}`);+ }+ ),+ textMatch(+ /<mention-agent\b[^>]*>(.*?)<\/mention-agent>/,+ [],+ (node, [, label]) => {+ const name = withoutTags(label);+ if (!name) {+ node.remove();+ return;+ }+ node.setTextContent(`@${name}`);+ }+ ),+ textMatch(+ /<mention-date\b([^>]*?)(?:\/>|>(?:.*?)<\/mention-date>)/,+ [DateMentionNode],+ (node, [, attrs]) => $replaceWithDateMention(node, attributes(attrs))+ ),+ textMatch(+ /<(file|pdf|audio|video)\b([^>]*)>(.*?)<\/\1>/,+ [],+ (node, [, , attrs, caption]) => {+ const parsed = attributes(attrs);+ const url = parsed.src ?? parsed.url;+ const label = withoutTags(caption) || fileName(url) || 'Notion file';+ return $replaceWithLink(node, url, label);+ }+ ),+ textMatch(+ /<(?:file|pdf|audio|video)\b([^>]*?)\/?>/,+ [],+ (node, [, attrs]) => {+ const parsed = attributes(attrs);+ const url = parsed.src ?? parsed.url;+ return $replaceWithLink(node, url, fileName(url) ?? 'Notion file');+ }+ ),+ textMatch(+ /<unknown\b([^>]*?)\/?>/,+ [DocumentMentionNode],+ (node, [, attrs]) => {+ const parsed = attributes(attrs);+ if (findNotionPage(links, parsed.url ?? '')) {+ return $replaceWithPageReference(+ node,+ links,+ parsed,+ parsed.alt ?? ''+ );+ }+ if (!isWebUrl(parsed.url)) {+ node.remove();+ return;+ }+ return $replaceWithLink(+ node,+ parsed.url,+ parsed.alt?.trim() || 'Unsupported Notion content'+ );+ }+ ),+ textMatch(/<span\b([^>]*)>(.*?)<\/span>/, [], (node, [, attrs, inner]) => {+ node.setTextContent(inner);+ if (+ attributes(attrs).underline === 'true' &&+ !node.hasFormat('underline')+ ) {+ node.toggleFormat('underline');+ }+ return node;+ }),+ // Block colors and toggle-heading markers: `# Title {toggle="true"}`.+ textMatch(+ /[ \t]*\{(?:(?:toggle|color)\s*=\s*"[^"]*"[ \t]*)+\}[ \t]*$/,+ [],+ (node) => {+ node.remove();+ }+ ),+ // Anything Notion-specific left over keeps its text but loses its tag.+ textMatch(+ /<\/?(?:mention-[\w-]+|span|page|ancestor-\d+-page|database|data-source|unknown|file|pdf|audio|video|image|empty-block|table_of_contents|custom-block|synced_block(?:_reference)?|columns?|callout|details|summary|ancestor-path|colgroup|col|tr|td|th|table)\b[^>]*>/,+ [],+ (node) => {+ node.remove();+ }+ ),+ ];+}++const EMPTY_BLOCK: ElementTransformer = {+ dependencies: [],+ type: 'element',+ regExp: /^\s*<empty-block\s*\/?>\s*$/,+ export: () => null,+ // The paragraph stays, empty: Notion's way of spacing blocks apart.+ replace: () => {},+};++const DROPPED_BLOCK: ElementTransformer = {+ dependencies: [],+ type: 'element',+ regExp: /^\s*<(?:table_of_contents|custom-block)\b[^>]*\/?>\s*$/,+ export: () => null,+ replace: (parent) => {+ parent.remove();+ },+};++const IMAGE_BLOCK: ElementTransformer = {+ dependencies: [ImageNode],+ type: 'element',+ regExp: /^\s*<image\b([^>]*?)(?:\/>|>(.*?)<\/image>)\s*$/,+ export: () => null,+ replace: (parent, _children, [, attrs, caption]) => {+ const parsed = attributes(attrs);+ const url = parsed.src ?? parsed.source ?? parsed.url;+ if (!isWebUrl(url)) {+ parent.clear();+ parent.append($createTextNode(withoutTags(caption ?? '')));+ return;+ }+ parent.replace(+ $createImageNode({+ srcType: 'url',+ url,+ alt: withoutTags(caption ?? ''),+ id: '',+ width: 0,+ height: 0,+ })+ );+ },+};++const CONTAINER_TAGS = [+ 'details',+ 'callout',+ 'columns',+ 'column',+ 'synced_block',+ 'synced_block_reference',+ 'ancestor-path',+ 'table',+] as const;+type ContainerTag = (typeof CONTAINER_TAGS)[number];++const CONTAINER_OPEN = new RegExp(+ `^\\s*<(${CONTAINER_TAGS.join('|')})\\b([^>]*?)>`+);++function isContainerTag(tag: string): tag is ContainerTag {+ return CONTAINER_TAGS.some((candidate) => candidate === tag);+}++const FENCE = /^\s*(`{3,}|~{3,})/;++/**+ * The lines inside a container opened at `start`, up to its matching close+ * tag (nested containers of the same tag included), and what follows the+ * close tag on its line.+ */+function containerContent(+ lines: string[],+ start: number,+ tag: ContainerTag,+ rest: string+): { inner: string[]; end: number; after: string } {+ const tags = new RegExp(`<(/?)${tag}\\b[^>]*?(/?)>`, 'g');+ const inner: string[] = [];+ let depth = 1;+ let fence: string | undefined;+ let line = rest;+ for (let index = start; index < lines.length; ) {+ const marker = line.match(FENCE)?.[1];+ if (fence) {+ if (marker?.startsWith(fence)) fence = undefined;+ } else if (marker) {+ fence = marker;+ } else {+ for (const found of line.matchAll(tags)) {+ if (found[2]) continue;+ depth += found[1] ? -1 : 1;+ if (depth === 0) {+ const at = found.index ?? 0;+ inner.push(line.slice(0, at));+ return { inner, end: index, after: line.slice(at + found[0].length) };+ }+ }+ }+ inner.push(line);+ index += 1;+ line = lines[index] ?? '';+ }+ // Never closed: everything to the end of the page is its content.+ return { inner, end: lines.length - 1, after: '' };+}++function dedent(lines: string[]): string {+ return lines+ .map((line) => line.replace(/^(?:\t| {4})/, ''))+ .filter((line) => line.trim() !== '')+ .join('\n');+}++/** Callout content as quotes: runs of paragraphs share one quote, the icon leading. */+function $calloutNodes(+ nodes: LexicalNode[],+ icon: string | undefined+): LexicalNode[] {+ const output: LexicalNode[] = [];+ let quote: QuoteNode | undefined;+ let leading = icon && !isWebUrl(icon) ? `${icon} ` : undefined;+ for (const node of nodes) {+ if (!$isParagraphNode(node)) {+ quote = undefined;+ output.push(node);+ continue;+ }+ if (quote) {+ quote.append($createLineBreakNode());+ } else {+ quote = $createQuoteNode();+ output.push(quote);+ }+ if (leading) {+ quote.append($createTextNode(leading));+ leading = undefined;+ }+ quote.append(...node.getChildren());+ }+ return output;+}++function $tableNode(+ html: string,+ attrs: Attributes,+ $import: (markdown: string) => LexicalNode[]+): TableNode | undefined {+ const rows = [...html.matchAll(/<tr\b[^>]*>([\s\S]*?)<\/tr>/g)]+ .map(([, row]) =>+ [...row.matchAll(/<t[dh]\b[^>]*>([\s\S]*?)<\/t[dh]>/g)].map(+ ([, cell]) => cell+ )+ )+ .filter((cells) => cells.length > 0);+ const columns = Math.max(0, ...rows.map((cells) => cells.length));+ if (columns === 0) return undefined;+ const headerRow = attrs['header-row'] === 'true';+ const headerColumn = attrs['header-column'] === 'true';+ const table = $createTableNode();+ rows.forEach((cells, rowIndex) => {+ const row = $createTableRowNode();+ for (let column = 0; column < columns; column += 1) {+ const header =+ (headerRow && rowIndex === 0 ? TableCellHeaderStates.ROW : 0) |+ (headerColumn && column === 0 ? TableCellHeaderStates.COLUMN : 0);+ const cell = $createTableCellNode(header);+ const content = (cells[column] ?? '')+ .split(/<br\s*\/?>|\n/)+ .map((line) => line.trim())+ .filter((line) => line !== '')+ .join('\n');+ const nodes = content ? $import(content) : [];+ cell.append(...(nodes.length > 0 ? nodes : [$createParagraphNode()]));+ row.append(cell);+ }+ table.append(row);+ });+ return table;+}++function containerTransformer(+ $import: (markdown: string) => LexicalNode[]+): MultilineElementTransformer {+ return {+ dependencies: [TableNode, TableRowNode, TableCellNode, LineBreakNode],+ type: 'multiline-element',+ regExpStart: CONTAINER_OPEN,+ export: () => null,+ handleImportAfterStartMatch: ({+ lines,+ rootNode,+ startLineIndex,+ startMatch,+ }) => {+ const [opening, tag, rawAttrs] = startMatch;+ if (!isContainerTag(tag) || rawAttrs.endsWith('/')) return null;+ const attrs = attributes(rawAttrs);+ const rest = lines[startLineIndex].slice(+ (startMatch.index ?? 0) + opening.length+ );+ const { inner, end, after } = containerContent(+ lines,+ startLineIndex,+ tag,+ rest+ );+ const content = dedent(inner);+ const nodes = match(tag)+ .with('details', () => {+ const summary = content.match(/^\s*<summary>([\s\S]*?)<\/summary>/);+ const children = summary ? content.slice(summary[0].length) : content;+ return [+ ...(summary ? $import(summary[1].replaceAll('\n', ' ')) : []),+ ...$import(children.trim()),+ ];+ })+ .with('callout', () => $calloutNodes($import(content), attrs.icon))+ .with('table', () => {+ const table = $tableNode(content, attrs, $import);+ return table ? [table] : [];+ })+ .with('ancestor-path', () => [])+ .with(+ 'columns',+ 'column',+ 'synced_block',+ 'synced_block_reference',+ () => $import(content)+ )+ .exhaustive();+ rootNode.append(...nodes);+ if (after.trim()) rootNode.append(...$import(after.trim()));+ return [true, end];+ },+ replace: () => false,+ };+}++/**+ * Notion markdown with blank lines dropped (Notion separates blocks by line,+ * and spells an empty block `<empty-block/>`), indentation kept only where+ * it means nesting (list items, code), and to-dos the MCP server escapes+ * (`\[x\] Done`) restored.+ */+export function notionLines(markdown: string): string {+ const output: string[] = [];+ let fence: string | undefined;+ for (const raw of markdown.split('\n')) {+ const line = raw.replace(/\r$/, '');+ const marker = line.match(FENCE)?.[1];+ if (fence) {+ output.push(line);+ if (marker?.startsWith(fence)) fence = undefined;+ continue;+ }+ if (marker) {+ fence = marker;+ output.push(line);+ continue;+ }+ if (line.trim() === '') continue;+ const todo = line.match(/^(\s*)(?:-\s+)?\\\[([ xX])\\\](.*)$/);+ if (todo) {+ output.push(`${todo[1]}- [${todo[2].toLowerCase()}]${todo[3]}`);+ continue;+ }+ output.push(/^\s*(?:[-*+]|\d+[.)])\s/.test(line) ? line : line.trimStart());+ }+ return output.join('\n');+}++/** The transformers for Notion markdown whose references resolve through `links`. */+export function notionTransformers(links: NotionLinks): Transformer[] {+ const transformers: Transformer[] = [];+ const $import = (markdown: string): LexicalNode[] => {+ // A detached scratch container: the import clears whatever node it+ // writes into, so it can't write into the root being built.+ const scratch: ElementNode = $createParagraphNode();+ $convertFromMarkdownString(markdown, transformers, scratch, true);+ return scratch.getChildren();+ };+ transformers.push(+ containerTransformer($import),+ EMPTY_BLOCK,+ DROPPED_BLOCK,+ IMAGE_BLOCK,+ ...inlineTransformers(links),+ ...ALL_TRANSFORMERS+ );+ return transformers;+}diff --git a/packages/lexical-core/tests/notion-markdown.test.ts b/packages/lexical-core/tests/notion-markdown.test.tsnew file mode 100644index 0000000000..f9f6ce7fcf--- /dev/null+++ b/packages/lexical-core/tests/notion-markdown.test.ts@@ -0,0 +1,326 @@+import type {+ SerializedEditorState,+ SerializedElementNode,+ SerializedLexicalNode,+} from 'lexical';+import { describe, expect, it } from 'vitest';+import {+ notionMarkdownToLoroSnapshot,+ notionMarkdownToSerializedEditorState,+} from '../notion-markdown';++const LINKS = {+ pages: {+ '3fb607323e2e80629886fc0c028d50a7': {+ documentId: '0e5b6a2c-7c39-4a63-9a0c-5c1f2f6c0001',+ name: 'Roadmap',+ },+ },+ users: {+ '12345678123412341234123456789abc': {+ userId: 'macro|[email protected]',+ email: '[email protected]',+ },+ },+};++type Node = SerializedLexicalNode & Record<string, unknown>;++function children(node: SerializedLexicalNode): Node[] {+ return 'children' in node+ ? (node as SerializedElementNode).children.map((child) => child as Node)+ : [];+}++function blocks(state: SerializedEditorState): Node[] {+ return children(state.root);+}++function text(node: SerializedLexicalNode): string {+ if (node.type === 'linebreak') return '\n';+ const own = 'text' in node && typeof node.text === 'string' ? node.text : '';+ return own + children(node).map(text).join('');+}++function convert(markdown: string): Node[] {+ return blocks(notionMarkdownToSerializedEditorState(markdown, LINKS));+}++describe('Notion markdown to Lexical', () => {+ it('turns a mapped page mention into a document mention', () => {+ const [paragraph] = convert(+ 'See <mention-page url="https://www.notion.so/Roadmap-3fb607323e2e80629886fc0c028d50a7">Roadmap</mention-page>.'+ );+ expect(children(paragraph).map((node) => node.type)).toEqual([+ 'text',+ 'document-mention',+ 'text',+ ]);+ expect(children(paragraph)[1]).toMatchObject({+ documentId: '0e5b6a2c-7c39-4a63-9a0c-5c1f2f6c0001',+ documentName: 'Roadmap',+ blockName: 'md',+ });+ });++ it('turns a child page placeholder into a document mention', () => {+ const [paragraph] = convert(+ '<page url="https://www.notion.so/3fb60732-3e2e-8062-9886-fc0c028d50a7">Roadmap</page>'+ );+ expect(children(paragraph).map((node) => node.type)).toEqual([+ 'document-mention',+ ]);+ });++ it('links an unmapped page back to Notion', () => {+ const [paragraph] = convert(+ '<mention-page url="https://www.notion.so/1111111111111111111111111111aaaa">Elsewhere</mention-page>'+ );+ const [link] = children(paragraph);+ expect(link).toMatchObject({+ type: 'link',+ url: 'https://www.notion.so/1111111111111111111111111111aaaa',+ });+ expect(text(link)).toBe('Elsewhere');+ });++ it('turns a matched person into a user mention and anyone else into text', () => {+ const [paragraph] = convert(+ 'Ask <mention-user url="user://12345678-1234-1234-1234-123456789abc">Teo Nys</mention-user> or <mention-user url="user://00000000-0000-0000-0000-000000000000">Ghost</mention-user>'+ );+ expect(children(paragraph).map((node) => node.type)).toEqual([+ 'text',+ 'user-mention',+ 'text',+ ]);+ expect(children(paragraph)[1]).toMatchObject({+ userId: 'macro|[email protected]',+ email: '[email protected]',+ });+ expect(text(paragraph)).toBe('Ask or @Ghost');+ });++ it('turns dates and date ranges into date mentions', () => {+ const [single, range] = convert(+ 'Starts <mention-date start="2024-04-01"/>\nRuns <mention-date start="2024-04-01" end="2024-04-05"/>'+ );+ expect(children(single)[1]).toMatchObject({+ type: 'date-mention',+ date: '2024-04-01T12:00:00.000Z',+ displayFormat: 'April 1, 2024',+ });+ expect(children(range).map((node) => node.type)).toEqual([+ 'text',+ 'date-mention',+ 'text',+ 'date-mention',+ ]);+ });++ it('places a timed date in its time zone', () => {+ const [paragraph] = convert(+ '<mention-date start="2024-07-01" startTime="09:30" timeZone="America/New_York"/>'+ );+ expect(children(paragraph)[0]).toMatchObject({+ type: 'date-mention',+ date: '2024-07-01T13:30:00.000Z',+ });+ });++ it('builds a Lexical table from a Notion table', () => {+ const [table] = convert(`<table fit-page-width="true" header-row="true">+<colgroup>+<col width="379">+<col color="gray">+</colgroup>+<tr>+<td>Editor</td>+<td>Pricing</td>+</tr>+<tr>+<td>Forma | Pro<br>second line</td>+</tr>+</table>`);+ expect(table.type).toBe('table');+ const rows = children(table);+ expect(rows.map((row) => children(row).length)).toEqual([2, 2]);+ expect(children(rows[0])[0]).toMatchObject({ headerState: 1 });+ expect(children(rows[1])[0]).toMatchObject({ headerState: 0 });+ expect(children(children(rows[1])[0]).map(text)).toEqual([+ 'Forma | Pro',+ 'second line',+ ]);+ });++ it('flattens toggles into their summary and content, with check lists', () => {+ const nodes = convert(`<details color="gray">+<summary>Coming in Q2</summary>+ - [ ] Open task+ - [x] Closed task+ Note under the toggle+</details>`);+ expect(nodes.map((node) => node.type)).toEqual([+ 'paragraph',+ 'task-list',+ 'paragraph',+ ]);+ expect(text(nodes[0])).toBe('Coming in Q2');+ expect(children(nodes[1]).map((item) => item.checked)).toEqual([+ false,+ true,+ ]);+ });++ it('keeps nested toggles and their content in order', () => {+ const nodes = convert(`<details>+<summary>Outer</summary>+ <details>+ <summary>Inner</summary>+ Deep+ </details>+ After inner+</details>+After outer`);+ expect(nodes.map(text)).toEqual([+ 'Outer',+ 'Inner',+ 'Deep',+ 'After inner',+ 'After outer',+ ]);+ });++ it('turns a callout into a quote led by its icon', () => {+ const [quote] = convert(`<callout icon="🔢" color="gray_bg">+ Numbers are estimates in **weeks**.+ Second line.+</callout>`);+ expect(quote.type).toBe('quote');+ expect(text(quote)).toBe(+ '🔢 Numbers are estimates in weeks.\nSecond line.'+ );+ });++ it('flattens columns into their content in reading order', () => {+ const nodes = convert(`<columns>+ <column>+ Left+ </column>+ <column>+ Right+ </column>+</columns>`);+ expect(nodes.map((node) => [node.type, text(node)])).toEqual([+ ['paragraph', 'Left'],+ ['paragraph', 'Right'],+ ]);+ });++ it('turns block and inline equations into equation nodes', () => {+ const [block, inline] = convert('$$E=mc^2$$\nInline $x^2$ math');+ expect(children(block)[0]).toMatchObject({+ type: 'equation',+ equation: 'E=mc^2',+ inline: false,+ });+ expect(children(inline)[1]).toMatchObject({+ type: 'equation',+ equation: 'x^2',+ inline: true,+ });+ });++ it('turns images into image nodes and other files into links', () => {+ const nodes = convert(`+<image src="https://img.example/b.png">Photo</image>+<file src="https://files.example/plan.docx">Plan.docx</file>+<video src="https://video.example/demo.mp4"></video>`);+ expect(nodes.map((node) => node.type)).toEqual([+ 'image',+ 'image',+ 'paragraph',+ 'paragraph',+ ]);+ expect(nodes[1]).toMatchObject({ url: 'https://img.example/b.png' });+ expect(children(nodes[2])[0]).toMatchObject({+ type: 'link',+ url: 'https://files.example/plan.docx',+ });+ expect(text(nodes[3])).toBe('demo.mp4');+ });++ it('resolves unknown blocks and databases to mentions when mapped, else readable links', () => {+ const nodes =+ convert(`<unknown url="https://www.notion.so/parent#3fb607323e2e80629886fc0c028d50a7" alt="child_page"/>+<unknown url="https://www.notion.so/9999999999999999999999999999aaaa" alt="Bookmark"/>+<unknown alt="embed"/>+<database url="https://www.notion.so/2222222222222222222222222222aaaa" inline="true">Bugs</database>`);+ expect(+ nodes.map((node) => children(node).map((child) => child.type))+ ).toEqual([['document-mention'], ['link'], [], ['link']]);+ expect(nodes.map(text)).toEqual(['', 'Bookmark', '', 'Bugs']);+ });++ it('keeps span text, applies underline, and drops block attributes', () => {+ const [paragraph] = convert(+ '<span underline="true">Under</span> and <span color="red">red</span> text {color="blue"}'+ );+ expect(text(paragraph)).toBe('Under and red text');+ expect(children(paragraph)[0]).toMatchObject({ text: 'Under', format: 8 });+ });++ it('strips toggle markers from headings', () => {+ const [heading, child] = convert('# Plan {toggle="true"}\n\tChild text');+ expect(heading).toMatchObject({ type: 'heading', tag: 'h1' });+ expect(text(heading)).toBe('Plan');+ expect(text(child)).toBe('Child text');+ });++ it('keeps empty blocks, drops tables of contents and ancestor paths', () => {+ const nodes = convert(`<ancestor-path>+<ancestor-1-page url="https://www.notion.so/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" title="Parent"/>+</ancestor-path>+Before+<empty-block/>+<table_of_contents color="gray"/>+After`);+ expect(nodes.map((node) => [node.type, text(node)])).toEqual([+ ['paragraph', 'Before'],+ ['paragraph', ''],+ ['paragraph', 'After'],+ ]);+ });++ it('restores escaped to-dos but leaves code untouched', () => {+ const nodes = convert(+ '\\[x\\] Completed\n```text\n\\[x\\] example\n<page url="https://www.notion.so/x">literal</page>\n```'+ );+ expect(nodes[0].type).toBe('task-list');+ expect(children(nodes[0])[0]).toMatchObject({ checked: true });+ expect(nodes[1].type).toBe('custom-code');+ expect(text(nodes[1])).toBe(+ '\\[x\\] example\n<page url="https://www.notion.so/x">literal</page>'+ );+ });++ it('never leaves a Notion tag as text', () => {+ const nodes = convert(+ '<synced_block url="https://www.notion.so/s"><mention-database url="https://www.notion.so/d">Tasks</mention-database> <mention-agent url="agent://a">Bot</mention-agent> <custom-tag>kept</custom-tag></synced_block>'+ );+ expect(nodes.map(text)).toEqual([+ 'Tasks @Bot <custom-tag>kept</custom-tag>',+ ]);+ });+});++describe('Notion markdown to a Loro snapshot', () => {+ it('returns the snapshot with its Macro markdown, mentions included', async () => {+ const result = await notionMarkdownToLoroSnapshot(+ 'See <mention-page url="https://www.notion.so/3fb607323e2e80629886fc0c028d50a7">Roadmap</mention-page>',+ LINKS+ );+ expect(result.snapshot.byteLength).toBeGreaterThan(0);+ expect(result.markdown).toContain('<m-document-mention>');+ expect(result.markdown).toContain('0e5b6a2c-7c39-4a63-9a0c-5c1f2f6c0001');+ });+});diff --git a/services/document_upload_finalizer_handler/src/app/tests.rs b/services/document_upload_finalizer_handler/src/app/tests.rsindex 7ad266550d..e81a13bd7b 100644--- a/services/document_upload_finalizer_handler/src/app/tests.rs+++ b/services/document_upload_finalizer_handler/src/app/tests.rs@@ -6,6 +6,8 @@ use documents::domain::legacy_office_upgrade::{LegacyOfficeUpgradeRequest, UpgradedObjectDestination,};use documents::domain::models::DocumentError;+use documents::domain::notion_markdown::NotionMarkdown;+use documents::domain::ports::markdown::MarkdownSnapshot;use model::document::DocumentBasic;use model_owner::Owner;@@ -194,6 +196,17 @@ impl MarkdownInitializationPort for NoMarkdown {) -> Result<Vec<u8>, DocumentError> {Ok(Vec::new())}++ async fn initialize_existing_notion(+ &self,+ _document_id: &str,+ _notion: &NotionMarkdown,+ ) -> Result<MarkdownSnapshot, DocumentError> {+ Ok(MarkdownSnapshot {+ snapshot: Vec::new(),+ markdown: String::new(),+ })+ }}async fn handle(file_type: Option<&str>, content: DocumentContent, key: String) -> Vec<Call> {diff --git a/services/lexical-service/AGENTS.md b/services/lexical-service/AGENTS.mdindex 3d5dbbb031..ca1e4668ee 100644--- a/services/lexical-service/AGENTS.md+++ b/services/lexical-service/AGENTS.md@@ -14,7 +14,8 @@ comment/markdown rendering. Production runs on ECS behind the gateway atmiddleware.- `src/endpoints/`: one file per route (`plaintext.ts`, `search-text.ts`,`cognition-text.ts`/`cognition-v2.ts`, `markdown.ts`, `markdown-snapshot.ts`,- `html.ts`, `xml.ts`, `comment-mark.ts`, `comment-only-change.ts`,+ `notion-snapshot.ts`, `html.ts`, `xml.ts`, `comment-mark.ts`,+ `comment-only-change.ts`,`image-markdown.ts`, `agent-context.ts`, `agent-announcement.ts`,`extract-reply.ts`, `mentions.ts`).- `src/lib/`: conversion logic the endpoints call into (`editor.ts`,diff --git a/services/lexical-service/README.md b/services/lexical-service/README.mdindex 0673dfaf81..a4452c8fc6 100644--- a/services/lexical-service/README.md+++ b/services/lexical-service/README.md@@ -26,6 +26,12 @@ Returns the pre-processed document to be uses in the cognition service.## POST /snapshot/markdownAccepts `{ "markdown": "..." }` and returns an `application/octet-stream` Loro snapshot equivalent to the markdown block migration path.+## POST /snapshot/notion+Accepts `{ "markdown": "...", "links": { "pages": { "<notion id>": { "documentId", "name" } }, "users": { "<notion id>": { "userId", "email" } } } }`+with a page in Notion's enhanced markdown and returns `{ "snapshot": "<base64 Loro snapshot>", "markdown": "<Macro markdown>" }`.+Notion ids are dashless and lowercase. Mapped page and people references become document and user mentions;+the rest degrade to links or text (`packages/lexical-core/notion/`). Deploy this endpoint before the import backend that calls it.+## POST /image-markdownAccepts `{ "staticFileId": "<uuid>", "url": "https://...", "width": 1536, "height": 1024 }`and returns `{ "markdown": "..." }` built from the shared Lexical image node.diff --git a/services/lexical-service/src/endpoints/notion-snapshot.test.ts b/services/lexical-service/src/endpoints/notion-snapshot.test.tsnew file mode 100644index 0000000000..b66944e532--- /dev/null+++ b/services/lexical-service/src/endpoints/notion-snapshot.test.ts@@ -0,0 +1,61 @@+import '../polyfills/prism';+import { describe, expect, it } from 'bun:test';+import { LoroDoc } from 'loro-crdt';+import app from '../index';++const request = (body: unknown, authenticated = true) =>+ app.request(+ '/snapshot/notion',+ {+ method: 'POST',+ headers: {+ 'Content-Type': 'application/json',+ ...(authenticated ? { 'x-internal-auth-key': 'test-key' } : {}),+ },+ body: JSON.stringify(body),+ },+ { INTERNAL_AUTH_KEY: 'test-key' }+ );++const page = {+ markdown:+ '# Launch\nOwned by <mention-user url="user://12345678-1234-1234-1234-123456789abc">Teo</mention-user>, see <mention-page url="https://www.notion.so/Spec-3fb607323e2e80629886fc0c028d50a7">Spec</mention-page>',+ links: {+ pages: {+ '3fb607323e2e80629886fc0c028d50a7': {+ documentId: '0e5b6a2c-7c39-4a63-9a0c-5c1f2f6c0001',+ name: 'Spec',+ },+ },+ users: {+ '12345678123412341234123456789abc': {+ userId: 'macro|[email protected]',+ email: '[email protected]',+ },+ },+ },+};++describe('notion snapshot endpoint', () => {+ it('returns a loadable Loro snapshot and its Macro markdown', async () => {+ const response = await request(page);+ expect(response.status).toBe(200);+ const body: { snapshot: string; markdown: string } = await response.json();+ const doc = new LoroDoc();+ doc.import(Buffer.from(body.snapshot, 'base64'));+ expect(doc.toJSON()).toBeTruthy();+ expect(body.markdown).toContain('# Launch');+ expect(body.markdown).toContain('<m-user-mention>');+ expect(body.markdown).toContain(+ '"documentId":"0e5b6a2c-7c39-4a63-9a0c-5c1f2f6c0001"'+ );+ });++ it('requires internal authentication', async () => {+ expect((await request(page, false)).status).toBe(401);+ });++ it('rejects a request without links', async () => {+ expect((await request({ markdown: page.markdown })).status).toBe(400);+ });+});diff --git a/services/lexical-service/src/endpoints/notion-snapshot.ts b/services/lexical-service/src/endpoints/notion-snapshot.tsnew file mode 100644index 0000000000..5fa3e33fc6--- /dev/null+++ b/services/lexical-service/src/endpoints/notion-snapshot.ts@@ -0,0 +1,70 @@+import { notionMarkdownToLoroSnapshot } from '@macro-inc/lexical-core/notion-markdown';+import { OpenAPIRoute } from 'chanfana';+import type { Context } from 'hono';+import { z } from 'zod';+import { handleEndpointError } from '../lib/error-handler';+import { standardErrorResponses } from '../lib/schemas';++const notionSnapshotRequest = z.object({+ markdown: z.string(),+ links: z.object({+ pages: z.record(+ z.string(),+ z.object({ documentId: z.string(), name: z.string() })+ ),+ users: z.record(+ z.string(),+ z.object({ userId: z.string(), email: z.string() })+ ),+ }),+});++const notionSnapshotResponse = z.object({+ snapshot: z.string().describe('The Loro snapshot, base64-encoded'),+ markdown: z.string().describe('The same content as Macro markdown'),+});++export class NotionSnapshotEndpoint extends OpenAPIRoute {+ schema = {+ summary: 'Convert Notion markdown to a Loro snapshot',+ description:+ "Converts a page in Notion's enhanced markdown to Lexical nodes, resolving page and people references through `links` (keyed by dashless Notion id) into document and user mentions, assigns durable node ids, and returns the Loro snapshot with the same content as Macro markdown.",+ request: {+ body: {+ content: {+ 'application/json': {+ schema: notionSnapshotRequest,+ },+ },+ },+ },+ responses: {+ 200: {+ description: 'Successfully converted Notion markdown',+ content: {+ 'application/json': {+ schema: notionSnapshotResponse,+ },+ },+ },+ ...standardErrorResponses,+ },+ };++ async handle(c: Context) {+ // Validated outside the try so a bad body answers 400, not 500.+ const { body } = await this.getValidatedData<typeof this.schema>();+ try {+ const { snapshot, markdown } = await notionMarkdownToLoroSnapshot(+ body.markdown,+ body.links+ );+ return c.json({+ snapshot: Buffer.from(snapshot).toString('base64'),+ markdown,+ });+ } catch (error) {+ return handleEndpointError(error, c);+ }+ }+}diff --git a/services/lexical-service/src/index.ts b/services/lexical-service/src/index.tsindex 392a9afcbf..eb8c5ae788 100644--- a/services/lexical-service/src/index.ts+++ b/services/lexical-service/src/index.ts@@ -15,6 +15,7 @@ import { ImageMarkdownEndpoint } from './endpoints/image-markdown';import { MarkdownEndpoint } from './endpoints/markdown';import { MarkdownSnapshotEndpoint } from './endpoints/markdown-snapshot';import { MentionsEndpoint } from './endpoints/mentions';+import { NotionSnapshotEndpoint } from './endpoints/notion-snapshot';import { PlaintextEndpoint } from './endpoints/plaintext';import { SearchTextEndpoint } from './endpoints/search-text';import { XmlEndpoint } from './endpoints/xml';@@ -77,6 +78,7 @@ openapi.get('/comment-mark/:docId/:markId', CommentMarkEndpoint);openapi.post('/comment-only-change', CommentOnlyChangeEndpoint);openapi.get('/xml/:docId', XmlEndpoint);openapi.post('/snapshot/markdown', MarkdownSnapshotEndpoint);+openapi.post('/snapshot/notion', NotionSnapshotEndpoint);openapi.post('/html', HtmlEndpoint);openapi.post('/mentions', MentionsEndpoint);openapi.post('/extract-reply', ExtractReplyEndpoint);