Skip to content

The open alternative to Salesforce, designed for AI.

perf(rich-text): parse markdown with marked instead of BlockNote's jsdom pipeline (#27142)

Charles Bochetcdad8a8b3836
diff --git a/packages/twenty-server/jest-integration.config.ts b/packages/twenty-server/jest-integration.config.ts
index e231ab30ef5..6fbb386bd8a 100644
--- a/packages/twenty-server/jest-integration.config.ts
+++ b/packages/twenty-server/jest-integration.config.ts
@@ -42,7 +42,7 @@ const jestConfig: JestConfigWithTsJest = {
maxWorkers: 1,
// ESM-only deps (jsdom 29 and msw chains, ai, @ai-sdk/*, @workflow/serde) that swc must transform for jest
transformIgnorePatterns: [
- '/node_modules/(?!(.*/node_modules/)?(jsdom|html-encoding-sniffer|whatwg-encoding|@exodus|parse5|entities|tough-cookie|@csstools|@asamuzakjp|msw|@mswjs|until-async|@bundled-es-modules|@open-draft|strict-event-emitter|headers-polyfill|outvariant|is-node-process|path-to-regexp|statuses|cookie|digest-fetch|md5|email-reply-parser|ai|@ai-sdk|@workflow|htmlparser2|domhandler|domutils|dom-serializer|domelementtype|@faker-js)/)',
+ '/node_modules/(?!(.*/node_modules/)?(jsdom|html-encoding-sniffer|whatwg-encoding|@exodus|parse5|entities|tough-cookie|@csstools|@asamuzakjp|msw|@mswjs|until-async|@bundled-es-modules|@open-draft|strict-event-emitter|headers-polyfill|outvariant|is-node-process|path-to-regexp|statuses|cookie|digest-fetch|md5|email-reply-parser|ai|@ai-sdk|@workflow|htmlparser2|marked|domhandler|domutils|dom-serializer|domelementtype|@faker-js)/)',
],
transform: {
'^.+\\.(t|j|mj)s$': [
diff --git a/packages/twenty-server/jest.config.mjs b/packages/twenty-server/jest.config.mjs
index e8b8648f5db..27312dfc7f0 100644
--- a/packages/twenty-server/jest.config.mjs
+++ b/packages/twenty-server/jest.config.mjs
@@ -17,7 +17,7 @@ const jestConfig = {
setupFilesAfterEnv: ['./setupTests.ts'],
transformIgnorePatterns: [
// ESM-only deps jest's CJS runtime cannot load; (.*/node_modules/)? also matches nested copies like sanitize-html's htmlparser2
- '/node_modules/(?!(.*/node_modules/)?(file-type|@file-type|strtok3|token-types|@borewit|@tokenizer|uint8array-extras|read-next-line|digest-fetch|md5|js-sha256|js-sha512|base-64|charenc|crypt|email-reply-parser|jsdom|html-encoding-sniffer|whatwg-encoding|@exodus|parse5|entities|tough-cookie|@csstools|@asamuzakjp|graphql-upload|fs-capacitor|e2b|@e2b|chalk|ai|@ai-sdk|@workflow|htmlparser2|domhandler|domutils|dom-serializer|domelementtype|@faker-js|twenty-oxlint-rules)/)',
+ '/node_modules/(?!(.*/node_modules/)?(file-type|@file-type|strtok3|token-types|@borewit|@tokenizer|uint8array-extras|read-next-line|digest-fetch|md5|js-sha256|js-sha512|base-64|charenc|crypt|email-reply-parser|jsdom|html-encoding-sniffer|whatwg-encoding|@exodus|parse5|entities|tough-cookie|@csstools|@asamuzakjp|graphql-upload|fs-capacitor|e2b|@e2b|chalk|ai|@ai-sdk|@workflow|htmlparser2|marked|domhandler|domutils|dom-serializer|domelementtype|@faker-js|twenty-oxlint-rules)/)',
],
testRegex: '.*\\.spec\\.ts$',
transform: {
diff --git a/packages/twenty-server/package.json b/packages/twenty-server/package.json
index 4d29d78a291..00fce477230 100644
--- a/packages/twenty-server/package.json
+++ b/packages/twenty-server/package.json
@@ -36,6 +36,7 @@
"@aws-sdk/lib-storage": "3.1001.0",
"@aws-sdk/s3-request-presigner": "3.1001.0",
"@azure/msal-node": "^5.2.3",
+ "@blocknote/core": "^0.51.4",
"@blocknote/server-util": "^0.51.4",
"@clickhouse/client": "^1.18.1",
"@e2b/code-interpreter": "^2.6.0",
@@ -134,6 +135,7 @@
"lodash.snakecase": "4.1.1",
"lodash.uniq": "^4.5.0",
"lodash.uniqby": "^4.7.0",
+ "marked": "^17.0.1",
"microdiff": "1.4.0",
"mrmime": "^2.0.1",
"ms": "2.1.3",
diff --git a/packages/twenty-server/src/engine/core-modules/record-transformer/utils/__tests__/convert-markdown-to-blocknote-blocks.util.spec.ts b/packages/twenty-server/src/engine/core-modules/record-transformer/utils/__tests__/convert-markdown-to-blocknote-blocks.util.spec.ts
new file mode 100644
index 00000000000..7ffa3120cb1
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/record-transformer/utils/__tests__/convert-markdown-to-blocknote-blocks.util.spec.ts
@@ -0,0 +1,396 @@
+import { convertMarkdownToBlocknoteBlocks } from 'src/engine/core-modules/record-transformer/utils/convert-markdown-to-blocknote-blocks.util';
+
+const DEFAULT_PROPS = {
+ backgroundColor: 'default',
+ textColor: 'default',
+ textAlignment: 'left',
+};
+
+const convertWithoutIds = (markdown: string) =>
+ JSON.parse(
+ JSON.stringify(convertMarkdownToBlocknoteBlocks(markdown), (key, value) =>
+ key === 'id' ? undefined : value,
+ ),
+ );
+
+const text = (value: string, styles: Record<string, true> = {}) => ({
+ type: 'text',
+ text: value,
+ styles,
+});
+
+describe('convertMarkdownToBlocknoteBlocks', () => {
+ it('should convert headings and inline styles', () => {
+ expect(
+ convertWithoutIds(
+ '## Title\n\n**bold *both*** ~~gone~~ `code` [link **x**](https://example.com)',
+ ),
+ ).toEqual([
+ {
+ type: 'heading',
+ props: { ...DEFAULT_PROPS, level: 2, isToggleable: false },
+ content: [text('Title')],
+ children: [],
+ },
+ {
+ type: 'paragraph',
+ props: DEFAULT_PROPS,
+ content: [
+ text('bold ', { bold: true }),
+ text('both', { bold: true, italic: true }),
+ text(' '),
+ text('gone', { strike: true }),
+ text(' '),
+ text('code', { code: true }),
+ text(' '),
+ {
+ type: 'link',
+ href: 'https://example.com',
+ content: [text('link '), text('x', { bold: true })],
+ },
+ ],
+ children: [],
+ },
+ ]);
+ });
+
+ it('should turn soft and hard line breaks into newlines', () => {
+ expect(convertWithoutIds('first \nsecond\nthird')).toEqual([
+ {
+ type: 'paragraph',
+ props: DEFAULT_PROPS,
+ content: [text('first\nsecond\nthird')],
+ children: [],
+ },
+ ]);
+ });
+
+ it('should convert ordered, nested and task lists', () => {
+ expect(
+ convertWithoutIds('3. three\n4. four\n - nested\n\n- [x] done'),
+ ).toEqual([
+ {
+ type: 'numberedListItem',
+ props: { ...DEFAULT_PROPS, start: 3 },
+ content: [text('three')],
+ children: [],
+ },
+ {
+ type: 'numberedListItem',
+ props: DEFAULT_PROPS,
+ content: [text('four')],
+ children: [
+ {
+ type: 'bulletListItem',
+ props: DEFAULT_PROPS,
+ content: [text('nested')],
+ children: [],
+ },
+ ],
+ },
+ {
+ type: 'checkListItem',
+ props: { ...DEFAULT_PROPS, checked: true },
+ content: [text('done')],
+ children: [],
+ },
+ ]);
+ });
+
+ it('should convert unchecked task items', () => {
+ expect(convertWithoutIds('- [ ] todo')).toEqual([
+ {
+ type: 'checkListItem',
+ props: { ...DEFAULT_PROPS, checked: false },
+ content: [text('todo')],
+ children: [],
+ },
+ ]);
+ });
+
+ it('should only nest under an ordered item when indented past its marker', () => {
+ const blockTypes = (markdown: string) =>
+ convertMarkdownToBlocknoteBlocks(markdown).map((block) => ({
+ type: block.type,
+ childTypes: block.children?.map((child) => child.type),
+ }));
+
+ expect(blockTypes('1. first\n - two spaces')).toEqual([
+ { type: 'numberedListItem', childTypes: [] },
+ { type: 'bulletListItem', childTypes: [] },
+ ]);
+ expect(blockTypes('1. first\n - three spaces')).toEqual([
+ { type: 'numberedListItem', childTypes: ['bulletListItem'] },
+ ]);
+ });
+
+ it('should turn bare urls into links', () => {
+ expect(convertWithoutIds('see https://example.com')[0].content).toEqual([
+ text('see '),
+ {
+ type: 'link',
+ href: 'https://example.com',
+ content: [text('https://example.com')],
+ },
+ ]);
+ });
+
+ it('should keep only the code style on inline code, even inside a link', () => {
+ expect(
+ convertWithoutIds('**bold `code`** [link `code`](https://example.com)')[0]
+ .content,
+ ).toEqual([
+ text('bold ', { bold: true }),
+ text('code', { code: true }),
+ text(' '),
+ {
+ type: 'link',
+ href: 'https://example.com',
+ content: [text('link ')],
+ },
+ text('code', { code: true }),
+ ]);
+ });
+
+ it('should turn images inside list items into image children', () => {
+ const [listItem] = convertWithoutIds(
+ '- item ![logo](https://example.com/logo.png)',
+ );
+
+ expect(listItem.content).toEqual([text('item ')]);
+ expect(listItem.children).toEqual([
+ expect.objectContaining({
+ type: 'image',
+ props: expect.objectContaining({
+ name: 'logo',
+ url: 'https://example.com/logo.png',
+ }),
+ }),
+ ]);
+ });
+
+ it('should drop urls the rich text validator would reject', () => {
+ expect(
+ convertWithoutIds(
+ '[click](javascript:alert%281%29) ![pixel](javascript:alert%281%29) [ok](https://example.com)',
+ ),
+ ).toEqual([
+ {
+ type: 'paragraph',
+ props: DEFAULT_PROPS,
+ content: [
+ text('click pixel '),
+ {
+ type: 'link',
+ href: 'https://example.com',
+ content: [text('ok')],
+ },
+ ],
+ children: [],
+ },
+ ]);
+ });
+
+ it('should keep quote content in source order', () => {
+ expect(convertWithoutIds('> before\n>\n> - middle\n>\n> after')).toEqual([
+ {
+ type: 'quote',
+ props: { backgroundColor: 'default', textColor: 'default' },
+ content: [text('before\nmiddle\nafter')],
+ children: [],
+ },
+ ]);
+ });
+
+ it('should turn images nested in links into image blocks', () => {
+ expect(
+ convertWithoutIds(
+ '[![logo](https://example.com/logo.png)](https://example.com)',
+ ),
+ ).toEqual([
+ {
+ type: 'image',
+ props: {
+ textAlignment: 'left',
+ backgroundColor: 'default',
+ name: 'logo',
+ url: 'https://example.com/logo.png',
+ caption: '',
+ showPreview: true,
+ },
+ children: [],
+ },
+ ]);
+ });
+
+ it('should keep text and nested images in source order', () => {
+ expect(
+ convertWithoutIds(
+ '**before ![logo](https://example.com/logo.png) after**',
+ ).map((block: { type: string; content?: unknown }) => [
+ block.type,
+ block.content,
+ ]),
+ ).toEqual([
+ ['paragraph', [text('before ', { bold: true })]],
+ ['image', undefined],
+ ['paragraph', [text(' after', { bold: true })]],
+ ]);
+ });
+
+ it('should keep quoted code and images', () => {
+ expect(
+ convertWithoutIds(
+ '> code:\n>\n> ```\n> x = 1\n> ```\n>\n> ![logo](https://example.com/logo.png)',
+ ).map((block: { type: string; content?: unknown }) => [
+ block.type,
+ block.content,
+ ]),
+ ).toEqual([
+ ['quote', [text('code:\nx = 1')]],
+ ['image', undefined],
+ ]);
+ });
+
+ it('should convert tables with their column alignment', () => {
+ const cellProps = {
+ colspan: 1,
+ rowspan: 1,
+ backgroundColor: 'default',
+ textColor: 'default',
+ };
+
+ expect(
+ convertWithoutIds('| Plan | Seats |\n|---|--:|\n| Pro | 10 |'),
+ ).toEqual([
+ {
+ type: 'table',
+ props: { textColor: 'default' },
+ content: {
+ type: 'tableContent',
+ columnWidths: [null, null],
+ headerRows: 1,
+ rows: [
+ {
+ cells: [
+ {
+ type: 'tableCell',
+ content: [text('Plan')],
+ props: { ...cellProps, textAlignment: 'left' },
+ },
+ {
+ type: 'tableCell',
+ content: [text('Seats')],
+ props: { ...cellProps, textAlignment: 'right' },
+ },
+ ],
+ },
+ {
+ cells: [
+ {
+ type: 'tableCell',
+ content: [text('Pro')],
+ props: { ...cellProps, textAlignment: 'left' },
+ },
+ {
+ type: 'tableCell',
+ content: [text('10')],
+ props: { ...cellProps, textAlignment: 'right' },
+ },
+ ],
+ },
+ ],
+ },
+ children: [],
+ },
+ ]);
+ });
+
+ it('should split a paragraph around images', () => {
+ expect(
+ convertWithoutIds('before ![logo](https://example.com/logo.png) after'),
+ ).toEqual([
+ {
+ type: 'paragraph',
+ props: DEFAULT_PROPS,
+ content: [text('before ')],
+ children: [],
+ },
+ {
+ type: 'image',
+ props: {
+ textAlignment: 'left',
+ backgroundColor: 'default',
+ name: 'logo',
+ url: 'https://example.com/logo.png',
+ caption: '',
+ showPreview: true,
+ },
+ children: [],
+ },
+ {
+ type: 'paragraph',
+ props: DEFAULT_PROPS,
+ content: [text(' after')],
+ children: [],
+ },
+ ]);
+ });
+
+ it('should convert quotes, code blocks and dividers', () => {
+ expect(
+ convertWithoutIds('> first\n>\n> second\n\n```\nplain\n```\n\n---'),
+ ).toEqual([
+ {
+ type: 'quote',
+ props: { backgroundColor: 'default', textColor: 'default' },
+ content: [text('first\nsecond')],
+ children: [],
+ },
+ {
+ type: 'codeBlock',
+ props: { language: 'text' },
+ content: [text('plain')],
+ children: [],
+ },
+ { type: 'divider', props: {}, children: [] },
+ ]);
+ });
+
+ it('should keep raw html as literal text', () => {
+ expect(
+ convertWithoutIds('Hello <b>world</b>\n\n<div>\n<p>raw</p>\n</div>'),
+ ).toEqual([
+ {
+ type: 'paragraph',
+ props: DEFAULT_PROPS,
+ content: [text('Hello <b>world</b>')],
+ children: [],
+ },
+ {
+ type: 'paragraph',
+ props: DEFAULT_PROPS,
+ content: [text('<div>\n<p>raw</p>\n</div>')],
+ children: [],
+ },
+ ]);
+ });
+
+ it('should return a single empty paragraph for empty markdown', () => {
+ expect(convertWithoutIds('')).toEqual([
+ { type: 'paragraph', props: DEFAULT_PROPS, content: [], children: [] },
+ ]);
+ });
+
+ it('should give every block a unique id', () => {
+ const blocks = convertMarkdownToBlocknoteBlocks('- a\n - b\n\nc');
+ const nestedBlock = blocks[0].children?.[0];
+
+ expect(nestedBlock).toBeDefined();
+
+ const ids = [blocks[0].id, nestedBlock?.id, blocks[1].id];
+
+ expect(ids.every((id) => typeof id === 'string')).toBe(true);
+ expect(new Set(ids).size).toBe(3);
+ });
+});
diff --git a/packages/twenty-server/src/engine/core-modules/record-transformer/utils/__tests__/transform-rich-text.util.spec.ts b/packages/twenty-server/src/engine/core-modules/record-transformer/utils/__tests__/transform-rich-text.util.spec.ts
new file mode 100644
index 00000000000..0caf5630b9f
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/record-transformer/utils/__tests__/transform-rich-text.util.spec.ts
@@ -0,0 +1,60 @@
+import { transformRichTextValue } from 'src/engine/core-modules/record-transformer/utils/transform-rich-text.util';
+
+const BLOCKNOTE_VALUE = JSON.stringify([
+ { id: '1', type: 'paragraph', content: [{ type: 'text', text: 'Hello' }] },
+]);
+
+const TIPTAP_VALUE = JSON.stringify({
+ type: 'doc',
+ content: [
+ {
+ type: 'paragraph',
+ content: [{ type: 'text', text: 'Hello', marks: [{ type: 'bold' }] }],
+ },
+ ],
+});
+
+const blockTypesOf = (blocknote: string | null | undefined) =>
+ JSON.parse(blocknote ?? '[]').map((block: { type: string }) => block.type);
+
+describe('transformRichTextValue', () => {
+ it('should build blocknote from markdown when only markdown is provided', async () => {
+ const result = await transformRichTextValue({
+ markdown: '# Title\n\n- item',
+ });
+
+ expect(result.markdown).toBe('# Title\n\n- item');
+ expect(blockTypesOf(result.blocknote)).toEqual([
+ 'heading',
+ 'bulletListItem',
+ ]);
+ });
+
+ it('should keep both values untouched when both are provided', async () => {
+ const result = await transformRichTextValue({
+ blocknote: BLOCKNOTE_VALUE,
+ markdown: 'Hello',
+ });
+
+ expect(result).toEqual({ markdown: 'Hello', blocknote: BLOCKNOTE_VALUE });
+ });
+
+ it('should return nulls for an empty value', async () => {
+ const result = await transformRichTextValue({
+ blocknote: '',
+ markdown: null,
+ });
+
+ expect(result).toEqual({ markdown: null, blocknote: null });
+ });
+
+ it('should rebuild blocknote from tiptap content', async () => {
+ const result = await transformRichTextValue({ blocknote: TIPTAP_VALUE });
+
+ expect(result.markdown).toBe('**Hello**');
+ expect(JSON.parse(result.blocknote ?? '[]')[0]).toMatchObject({
+ type: 'paragraph',
+ content: [{ type: 'text', text: 'Hello', styles: { bold: true } }],
+ });
+ });
+});
diff --git a/packages/twenty-server/src/engine/core-modules/record-transformer/utils/convert-markdown-inline-tokens.util.ts b/packages/twenty-server/src/engine/core-modules/record-transformer/utils/convert-markdown-inline-tokens.util.ts
new file mode 100644
index 00000000000..c27c057653e
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/record-transformer/utils/convert-markdown-inline-tokens.util.ts
@@ -0,0 +1,177 @@
+import type { DefaultStyleSchema, Link, StyledText } from '@blocknote/core';
+import { type Token, type Tokens } from 'marked';
+import { isDefined, isNonEmptyArray, isSafeUrl } from 'twenty-shared/utils';
+
+type InlineStyle = 'bold' | 'italic' | 'strike' | 'code';
+
+export type InlineContent =
+ | StyledText<DefaultStyleSchema>
+ | Link<DefaultStyleSchema>;
+
+type InlineRun = {
+ text: string;
+ styles: InlineStyle[];
+ href?: string;
+};
+
+const STYLE_BY_TOKEN_TYPE: Record<string, InlineStyle> = {
+ strong: 'bold',
+ em: 'italic',
+ del: 'strike',
+};
+
+export const getChildTokens = (token: Token): Token[] =>
+ 'tokens' in token && Array.isArray(token.tokens) ? token.tokens : [];
+
+export const isImageToken = (token: Token): token is Tokens.Image =>
+ token.type === 'image' && isSafeUrl(token.href);
+
+export const splitAroundImages = (token: Token): Token[] => {
+ const childTokens = getChildTokens(token);
+
+ if (isImageToken(token) || !isNonEmptyArray(childTokens)) {
+ return [token];
+ }
+
+ const parts: Token[] = [];
+ let pendingChildTokens: Token[] = [];
+
+ const flushChildTokens = () => {
+ if (pendingChildTokens.length > 0) {
+ const partToken: Tokens.Generic = {
+ ...token,
+ tokens: pendingChildTokens,
+ };
+
+ parts.push(partToken);
+ pendingChildTokens = [];
+ }
+ };
+
+ for (const childPart of childTokens.flatMap(splitAroundImages)) {
+ if (isImageToken(childPart)) {
+ flushChildTokens();
+ parts.push(childPart);
+ } else {
+ pendingChildTokens.push(childPart);
+ }
+ }
+
+ flushChildTokens();
+
+ return parts;
+};
+
+const collectInlineRuns = ({
+ tokens,
+ styles,
+ href,
+ runs,
+}: {
+ tokens: Token[];
+ styles: InlineStyle[];
+ href?: string;
+ runs: InlineRun[];
+}): void => {
+ for (const token of tokens) {
+ const style = STYLE_BY_TOKEN_TYPE[token.type];
+
+ if (isDefined(style)) {
+ collectInlineRuns({
+ tokens: getChildTokens(token),
+ styles: [...styles, style],
+ href,
+ runs,
+ });
+ continue;
+ }
+
+ switch (token.type) {
+ case 'text':
+ if (isNonEmptyArray(token.tokens)) {
+ collectInlineRuns({ tokens: token.tokens, styles, href, runs });
+ } else {
+ runs.push({ text: token.text, styles, href });
+ }
+ break;
+ case 'escape':
+ runs.push({ text: token.text, styles, href });
+ break;
+ case 'codespan':
+ runs.push({ text: token.text, styles: ['code'] });
+ break;
+ case 'br':
+ runs.push({ text: '\n', styles, href });
+ break;
+ case 'link':
+ collectInlineRuns({
+ tokens: getChildTokens(token),
+ styles,
+ href: isSafeUrl(token.href) ? token.href : href,
+ runs,
+ });
+ break;
+ case 'image':
+ runs.push({
+ text: token.text.length > 0 ? token.text : token.href,
+ styles,
+ href: isSafeUrl(token.href) ? token.href : href,
+ });
+ break;
+ case 'checkbox':
+ break;
+ default:
+ runs.push({ text: token.raw, styles, href });
+ }
+ }
+};
+
+const isSameRunFormat = (firstRun: InlineRun, secondRun: InlineRun) =>
+ firstRun.href === secondRun.href &&
+ firstRun.styles.length === secondRun.styles.length &&
+ firstRun.styles.every((style) => secondRun.styles.includes(style));
+
+export const convertMarkdownInlineTokens = (
+ tokens: Token[],
+): InlineContent[] => {
+ const runs: InlineRun[] = [];
+
+ collectInlineRuns({ tokens, styles: [], runs });
+
+ const mergedRuns: InlineRun[] = [];
+
+ for (const run of runs) {
+ const lastRun = mergedRuns[mergedRuns.length - 1];
+
+ if (run.text.length === 0) {
+ continue;
+ }
+
+ if (isDefined(lastRun) && isSameRunFormat(lastRun, run)) {
+ lastRun.text += run.text;
+ } else {
+ mergedRuns.push({ ...run });
+ }
+ }
+
+ const content: InlineContent[] = [];
+
+ for (const run of mergedRuns) {
+ const styledText: StyledText<DefaultStyleSchema> = {
+ type: 'text',
+ text: run.text,
+ styles: Object.fromEntries(run.styles.map((style) => [style, true])),
+ };
+ const lastContent = content[content.length - 1];
+
+ if (!isDefined(run.href)) {
+ content.push(styledText);
+ } else if (lastContent?.type === 'link' && lastContent.href === run.href) {
+ lastContent.content.push(styledText);
+ } else {
+ content.push({ type: 'link', href: run.href, content: [styledText] });
+ }
+ }
+
+ return content;
+};
diff --git a/packages/twenty-server/src/engine/core-modules/record-transformer/utils/convert-markdown-to-blocknote-blocks.util.ts b/packages/twenty-server/src/engine/core-modules/record-transformer/utils/convert-markdown-to-blocknote-blocks.util.ts
new file mode 100644
index 00000000000..8955d51c2e1
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/record-transformer/utils/convert-markdown-to-blocknote-blocks.util.ts
@@ -0,0 +1,281 @@
+import { randomUUID } from 'crypto';
+
+import type { PartialBlock } from '@blocknote/core';
+import { Lexer, type Token, type Tokens } from 'marked';
+import { isNonEmptyArray } from 'twenty-shared/utils';
+
+import {
+ convertMarkdownInlineTokens,
+ getChildTokens,
+ type InlineContent,
+ isImageToken,
+ splitAroundImages,
+} from 'src/engine/core-modules/record-transformer/utils/convert-markdown-inline-tokens.util';
+
+const DEFAULT_BLOCK_PROPS = {
+ backgroundColor: 'default',
+ textColor: 'default',
+ textAlignment: 'left',
+} as const;
+
+const createBlock = (block: PartialBlock): PartialBlock => ({
+ id: randomUUID(),
+ ...block,
+ children: block.children ?? [],
+});
+
+const isBlankContent = (content: InlineContent[]) =>
+ content.every((item) => item.type === 'text' && item.text.trim() === '');
+
+const LINE_BREAK_TOKEN: Tokens.Br = { type: 'br', raw: '\n' };
+
+const isListToken = (token: Token): token is Tokens.List =>
+ token.type === 'list';
+
+const isTableToken = (token: Token): token is Tokens.Table =>
+ token.type === 'table';
+
+const convertImage = (image: Tokens.Image): PartialBlock =>
+ createBlock({
+ type: 'image',
+ props: {
+ textAlignment: 'left',
+ backgroundColor: 'default',
+ name: image.text,
+ url: image.href,
+ caption: '',
+ showPreview: true,
+ },
+ });
+
+const convertParagraph = (tokens: Token[]): PartialBlock[] => {
+ const blocks: PartialBlock[] = [];
+ let pendingTokens: Token[] = [];
+
+ const flushParagraph = () => {
+ const content = convertMarkdownInlineTokens(pendingTokens);
+
+ if (!isBlankContent(content)) {
+ blocks.push(
+ createBlock({
+ type: 'paragraph',
+ props: { ...DEFAULT_BLOCK_PROPS },
+ content,
+ }),
+ );
+ }
+
+ pendingTokens = [];
+ };
+
+ for (const token of tokens.flatMap(splitAroundImages)) {
+ if (!isImageToken(token)) {
+ pendingTokens.push(token);
+ continue;
+ }
+
+ flushParagraph();
+ blocks.push(convertImage(token));
+ }
+
+ flushParagraph();
+
+ return blocks;
+};
+
+const convertListItem = ({
+ list,
+ item,
+ index,
+}: {
+ list: Tokens.List;
+ item: Tokens.ListItem;
+ index: number;
+}): PartialBlock => {
+ const tokens = item.tokens.filter(
+ (token) => token.type !== 'checkbox' && token.type !== 'space',
+ );
+ const [firstToken, ...otherTokens] = tokens;
+ const hasInlineFirstToken =
+ firstToken?.type === 'text' || firstToken?.type === 'paragraph';
+
+ const inlineParts = hasInlineFirstToken
+ ? (firstToken.tokens ?? []).flatMap(splitAroundImages)
+ : [];
+ const content = convertMarkdownInlineTokens(
+ inlineParts.filter((token) => !isImageToken(token)),
+ );
+ const children = [
+ ...inlineParts.filter(isImageToken).map(convertImage),
+ ...convertBlockTokens(hasInlineFirstToken ? otherTokens : tokens),
+ ];
+
+ if (item.task) {
+ return createBlock({
+ type: 'checkListItem',
+ props: { ...DEFAULT_BLOCK_PROPS, checked: item.checked === true },
+ content,
+ children,
+ });
+ }
+
+ if (list.ordered) {
+ const start = Number(list.start);
+
+ return createBlock({
+ type: 'numberedListItem',
+ props:
+ index === 0 && Number.isInteger(start) && start !== 1
+ ? { ...DEFAULT_BLOCK_PROPS, start }
+ : { ...DEFAULT_BLOCK_PROPS },
+ content,
+ children,
+ });
+ }
+
+ return createBlock({
+ type: 'bulletListItem',
+ props: { ...DEFAULT_BLOCK_PROPS },
+ content,
+ children,
+ });
+};
+
+const collectQuoteLines = (tokens: Token[]): Token[][] =>
+ tokens.flatMap((token): Token[][] => {
+ switch (token.type) {
+ case 'space':
+ return [];
+ case 'paragraph':
+ case 'text':
+ case 'heading':
+ return [getChildTokens(token)];
+ case 'list':
+ return isListToken(token)
+ ? token.items.flatMap((item) => collectQuoteLines(item.tokens))
+ : [];
+ case 'blockquote':
+ return collectQuoteLines(getChildTokens(token));
+ case 'code':
+ return [[{ type: 'text', raw: token.raw, text: token.text }]];
+ default:
+ return [[{ type: 'text', raw: token.raw, text: token.raw.trimEnd() }]];
+ }
+ });
+
+const convertTable = (table: Tokens.Table): PartialBlock => {
+ const toRow = (cells: Tokens.TableCell[]) => ({
+ cells: cells.map((cell, columnIndex) => ({
+ type: 'tableCell' as const,
+ content: convertMarkdownInlineTokens(cell.tokens),
+ props: {
+ colspan: 1,
+ rowspan: 1,
+ backgroundColor: 'default',
+ textColor: 'default',
+ textAlignment: table.align[columnIndex] ?? 'left',
+ },
+ })),
+ });
+
+ return createBlock({
+ type: 'table',
+ props: { textColor: 'default' },
+ content: {
+ type: 'tableContent',
+ columnWidths: table.header.map(() => undefined),
+ headerRows: 1,
+ rows: [toRow(table.header), ...table.rows.map(toRow)],
+ },
+ });
+};
+
+const convertBlockTokens = (tokens: Token[]): PartialBlock[] =>
+ tokens.flatMap((token): PartialBlock[] => {
+ switch (token.type) {
+ case 'space':
+ case 'def':
+ return [];
+ case 'heading':
+ return [
+ createBlock({
+ type: 'heading',
+ props: {
+ ...DEFAULT_BLOCK_PROPS,
+ level: token.depth,
+ isToggleable: false,
+ },
+ content: convertMarkdownInlineTokens(token.tokens ?? []),
+ }),
+ ];
+ case 'paragraph':
+ case 'text':
+ return convertParagraph(token.tokens ?? [token]);
+ case 'code':
+ return [
+ createBlock({
+ type: 'codeBlock',
+ props: { language: token.lang || 'text' },
+ content:
+ token.text.length > 0
+ ? [{ type: 'text', text: token.text, styles: {} }]
+ : [],
+ }),
+ ];
+ case 'hr':
+ return [createBlock({ type: 'divider', props: {} })];
+ case 'blockquote': {
+ const quoteLines = collectQuoteLines(token.tokens ?? []).map((line) =>
+ line.flatMap(splitAroundImages),
+ );
+ const quoteTextLines = quoteLines
+ .map((line) => line.filter((part) => !isImageToken(part)))
+ .filter(isNonEmptyArray);
+
+ return [
+ createBlock({
+ type: 'quote',
+ props: { backgroundColor: 'default', textColor: 'default' },
+ content: convertMarkdownInlineTokens(
+ quoteTextLines.flatMap((line, index) =>
+ index > 0 ? [LINE_BREAK_TOKEN, ...line] : line,
+ ),
+ ),
+ }),
+ ...quoteLines.flat().filter(isImageToken).map(convertImage),
+ ];
+ }
+ case 'list':
+ return isListToken(token)
+ ? token.items.map((item, index) =>
+ convertListItem({ list: token, item, index }),
+ )
+ : [];
+ case 'table':
+ return isTableToken(token) ? [convertTable(token)] : [];
+ default:
+ return [
+ createBlock({
+ type: 'paragraph',
+ props: { ...DEFAULT_BLOCK_PROPS },
+ content: [{ type: 'text', text: token.raw.trimEnd(), styles: {} }],
+ }),
+ ];
+ }
+ });
+
+export const convertMarkdownToBlocknoteBlocks = (
+ markdown: string,
+): PartialBlock[] => {
+ const blocks = convertBlockTokens(new Lexer({ gfm: true }).lex(markdown));
+
+ return blocks.length > 0
+ ? blocks
+ : [
+ createBlock({
+ type: 'paragraph',
+ props: { ...DEFAULT_BLOCK_PROPS },
+ content: [],
+ }),
+ ];
+};
diff --git a/packages/twenty-server/src/engine/core-modules/record-transformer/utils/transform-rich-text.util.ts b/packages/twenty-server/src/engine/core-modules/record-transformer/utils/transform-rich-text.util.ts
index 891175921ca..417242de973 100644
--- a/packages/twenty-server/src/engine/core-modules/record-transformer/utils/transform-rich-text.util.ts
+++ b/packages/twenty-server/src/engine/core-modules/record-transformer/utils/transform-rich-text.util.ts
@@ -7,26 +7,51 @@ import { convertTipTapBlocksToMarkdown, isDefined } from 'twenty-shared/utils';
import type { ServerBlockNoteEditor } from '@blocknote/server-util';
+import { convertMarkdownToBlocknoteBlocks } from 'src/engine/core-modules/record-transformer/utils/convert-markdown-to-blocknote-blocks.util';
+
// Reuse a single ServerBlockNoteEditor across all calls to avoid
// the cost of dynamic import resolution + instance creation (~90ms) on every transform.
-let cachedServerBlockNoteEditor: ServerBlockNoteEditor | null = null;
+let serverBlockNoteEditorPromise: Promise<ServerBlockNoteEditor> | null = null;
// SWC compiles import() to require() in CJS mode, which breaks ESM-only
// transitive dependencies in @blocknote/core. Native import() resolves
// the ESM bundle path where the full chain works.
const nativeImport = new Function('specifier', 'return import(specifier)');
-const getServerBlockNoteEditor = async (): Promise<ServerBlockNoteEditor> => {
- if (cachedServerBlockNoteEditor) {
- return cachedServerBlockNoteEditor;
- }
-
+const loadServerBlockNoteEditor = async (): Promise<ServerBlockNoteEditor> => {
const module = await nativeImport('@blocknote/server-util');
- const editor: ServerBlockNoteEditor = module.ServerBlockNoteEditor.create();
- cachedServerBlockNoteEditor = editor;
+ return module.ServerBlockNoteEditor.create();
+};
+
+const getServerBlockNoteEditor = (): Promise<ServerBlockNoteEditor> => {
+ serverBlockNoteEditorPromise ??= loadServerBlockNoteEditor().catch(
+ (error: unknown) => {
+ serverBlockNoteEditorPromise = null;
+ throw error;
+ },
+ );
- return editor;
+ return serverBlockNoteEditorPromise;
+};
+
+const convertMarkdownToBlocknote = (markdown: string): string =>
+ JSON.stringify(convertMarkdownToBlocknoteBlocks(markdown));
+
+// Patch: Handle cases where blocknote to markdown conversion fails for certain block types (custom/code blocks)
+// Todo : This may be resolved once the server-utils library is updated with proper conversion support - #947
+const convertBlocknoteToMarkdown = async (
+ blocknote: string,
+): Promise<string> => {
+ const serverBlockNoteEditor = await getServerBlockNoteEditor();
+
+ try {
+ return await serverBlockNoteEditor.blocksToMarkdownLossy(
+ JSON.parse(blocknote),
+ );
+ } catch {
+ return blocknote;
+ }
};
export const transformRichTextValue = async (
@@ -37,8 +62,6 @@ export const transformRichTextValue = async (
? richTextValueSchema.parse(richTextValue)
: richTextValue;
- const serverBlockNoteEditor = await getServerBlockNoteEditor();
-
const tipTapMarkdown = isDefined(parsedValue.blocknote)
? convertTipTapBlocksToMarkdown(parsedValue.blocknote)
: undefined;
@@ -46,36 +69,20 @@ export const transformRichTextValue = async (
if (isDefined(tipTapMarkdown)) {
return {
markdown: parsedValue.markdown || tipTapMarkdown,
- blocknote: JSON.stringify(
- await serverBlockNoteEditor.tryParseMarkdownToBlocks(tipTapMarkdown),
- ),
+ blocknote: convertMarkdownToBlocknote(tipTapMarkdown),
};
}
- // Patch: Handle cases where blocknote to markdown conversion fails for certain block types (custom/code blocks)
- // Todo : This may be resolved once the server-utils library is updated with proper conversion support - #947
- let convertedMarkdown: string | null = null;
-
- try {
- convertedMarkdown = isDefined(parsedValue.blocknote)
- ? await serverBlockNoteEditor.blocksToMarkdownLossy(
- JSON.parse(parsedValue.blocknote),
- )
- : null;
- } catch {
- convertedMarkdown = parsedValue.blocknote || null;
- }
-
- const convertedBlocknote = parsedValue.markdown
- ? JSON.stringify(
- await serverBlockNoteEditor.tryParseMarkdownToBlocks(
- parsedValue.markdown,
- ),
- )
- : null;
-
return {
- markdown: parsedValue.markdown || convertedMarkdown,
- blocknote: parsedValue.blocknote || convertedBlocknote,
+ markdown:
+ parsedValue.markdown ||
+ (isNonEmptyString(parsedValue.blocknote)
+ ? await convertBlocknoteToMarkdown(parsedValue.blocknote)
+ : null),
+ blocknote:
+ parsedValue.blocknote ||
+ (isNonEmptyString(parsedValue.markdown)
+ ? convertMarkdownToBlocknote(parsedValue.markdown)
+ : null),
};
};
diff --git a/yarn.lock b/yarn.lock
index 83dd695ee88..e7a08bcf89e 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -4603,7 +4603,7 @@ __metadata:
languageName: node
linkType: hard
-"@blocknote/core@npm:0.51.4":
+"@blocknote/core@npm:0.51.4, @blocknote/core@npm:^0.51.4":
version: 0.51.4
resolution: "@blocknote/core@npm:0.51.4"
dependencies:
@@ -50688,6 +50688,7 @@ __metadata:
"@aws-sdk/lib-storage": "npm:3.1001.0"
"@aws-sdk/s3-request-presigner": "npm:3.1001.0"
"@azure/msal-node": "npm:^5.2.3"
+ "@blocknote/core": "npm:^0.51.4"
"@blocknote/server-util": "npm:^0.51.4"
"@clickhouse/client": "npm:^1.18.1"
"@e2b/code-interpreter": "npm:^2.6.0"
@@ -50832,6 +50833,7 @@ __metadata:
lodash.snakecase: "npm:4.1.1"
lodash.uniq: "npm:^4.5.0"
lodash.uniqby: "npm:^4.7.0"
+ marked: "npm:^17.0.1"
microdiff: "npm:1.4.0"
mrmime: "npm:^2.0.1"
ms: "npm:2.1.3"