Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
refactor(web): highlight markdown code blocks with @pierre/highlights (#44522)
Stephen Zhou07a1debf7cea
diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 3ef00da27a2..2fabee50a6e 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -260,6 +260,9 @@ catalogs:'@orpc/tanstack-query':specifier: 1.15.7version: 1.15.7+ '@pierre/highlights':+ specifier: 0.2.0+ version: 0.2.0'@playwright/test':specifier: 1.64.1version: 1.64.1@@ -536,9 +539,6 @@ catalogs:fuse.js:specifier: 7.5.0version: 7.5.0- hast-util-to-jsx-runtime:- specifier: 2.3.6- version: 2.3.6hono:specifier: 4.13.13version: 4.13.13@@ -695,9 +695,6 @@ catalogs:sharp:specifier: 0.35.5version: 0.35.5- shiki:- specifier: 4.5.0- version: 4.5.0socket.io-client:specifier: 4.8.4version: 4.8.4@@ -1465,6 +1462,9 @@ importers:'@orpc/tanstack-query':specifier: 'catalog:'version: 1.15.7(@orpc/[email protected])(@tanstack/[email protected])+ '@pierre/highlights':+ specifier: 'catalog:'+ version: 0.2.0'@remixicon/react':specifier: 'catalog:'version: 4.9.0([email protected])@@ -1555,9 +1555,6 @@ importers:fuse.js:specifier: 'catalog:'version: 7.5.0- hast-util-to-jsx-runtime:- specifier: 'catalog:'- version: 2.3.6([email protected])html-entities:specifier: 'catalog:'version: 2.6.0@@ -1672,9 +1669,6 @@ importers:server-only:specifier: 'catalog:'version: 0.0.1- shiki:- specifier: 'catalog:'- version: 4.5.0socket.io-client:specifier: 'catalog:'version: 4.8.4([email protected])@@ -4114,6 +4108,9 @@ packages:resolution: {integrity: sha512-G2xbfWTrbqYRhR/vfeKNMQf/sS3/ti0iKJAAkRxHuXWZH/hne/dyIb9QKk5LuR6cJLARxsL6FMoAYmfspd74MQ==}engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}+ '@pierre/[email protected]':+ resolution: {integrity: sha512-hTpcOU1EzDAP4bYtEhxmMu2jwMvtkUydRUBUpWMSHTs3dw0IaC3Ld5ansMy18L0GE3b6Vfl2OGkLK+pWWpnBmw==}+'@pkgr/[email protected]':resolution: {integrity: sha512-SEeaJLb3qBNF/OaXnaR1NmmBbFYk1zC0ZH/52fATcRPLFg/p791YrcyFFy44Bo9sLaGuSuLp5Q6axbb/O+v/RA==}engines: {node: ^14.18.0 || >=16.0.0}@@ -4232,37 +4229,6 @@ packages:eslint:optional: true- '@shikijs/[email protected]':- resolution: {integrity: sha512-6CIgwq+bt2JZ3OSjb/nBkkwEf/CfpHcbam+Mg5Pllco7HAGRQ2/EFhZrrJtbyNTgKwknfVS+llsXGygiDGwJWw==}- engines: {node: '>=20'}-- '@shikijs/[email protected]':- resolution: {integrity: sha512-Ed6UdEj84LyYkwnxM5CvSMWAD+id9ExTYri0/+x41U9zrgNifPSS9lZm0JqBP0VfYI8yr8tY5nbNq9+p9R0kFA==}- engines: {node: '>=20'}-- '@shikijs/[email protected]':- resolution: {integrity: sha512-dKVmbJecVdB9recl8a/CU4jdpNXyjJQrIcesIkpS70s5HS02ohpnCSpMUbotPnEUzbAcLQ+S4hwZYbkBBiTcJw==}- engines: {node: '>=20'}-- '@shikijs/[email protected]':- resolution: {integrity: sha512-wwBTqwduXSz0X4v9Qa2EWeFaYJEw1clCncWjoRNQRbVDu3SPRTRWucnouYapKj3b8Q+sk/41Vbc4B5c8nuYzLA==}- engines: {node: '>=20'}-- '@shikijs/[email protected]':- resolution: {integrity: sha512-UM7ShlA8oz0OWxLLO8BmO9Ip6fVqdkt8yF0EH7nOCqHfrx1eEAzF0ZUVK7L+I8NEgeu1PEWSRlxaiPPeCb4iDg==}- engines: {node: '>=20'}-- '@shikijs/[email protected]':- resolution: {integrity: sha512-zGb87H3UBbpuI4yoh5UguB1XbU+2BDU3kflM7UfZ+Cy2hWo9lBtqjSVK20gmLY6ukYl040hAypGXMgud9vul8Q==}- engines: {node: '>=20'}-- '@shikijs/[email protected]':- resolution: {integrity: sha512-drtPrtD8N0DENyD2cyt6Ajy+IWsiCLgMxODwLTpvH1UsN6a9jf90tNTPjqwHvA+0Lzu9e0txk8DjfqH1p9UJRw==}- engines: {node: '>=20'}-- '@shikijs/[email protected]':- resolution: {integrity: sha512-83yeghZ2xxin3Nj8z1NMd/NCuca+gsYXswywDy5bHvwlWL8tpTQmzGeUuHd9FC3E/SBEMvzJRwWEOz5gGes9Qg==}-'@shuding/[email protected]':resolution: {integrity: sha512-3NgmNyH3l/Hv6EvsWJbsvpcpUba6R8IREQ83nH83cyakCw7uM1arZKNfHwv1Wz6jgqrF/j4x5ELvR6PnK9nTcA==}engines: {node: '>= 8.0.0'}@@ -6790,9 +6756,6 @@ packages:resolution: {integrity: sha512-3yTWghByc50aGS7JlGhk61SPenfE/p1oaFeNwkOOyrscaOkMGrcW9+Cy/QAIOBpZxP1yqDIzFMR0+Np0i0+usg==}- resolution: {integrity: sha512-OguPdidb+fbHQSU4Q4ZiLKnzWo8Wwsf5bZfbvu7//a9oTYoqD/fWpe96NuHkoS9h0ccGOTe0C4NGXdtS0iObOw==}-resolution: {integrity: sha512-zl6s8LwNyo1P9uw+XJGvZtdFF1GdAkOg8ujOw+4Pyb76874fLps4ueHXDhXWdk6YHQ6OgUtinliG7RsYvCbbBg==}@@ -7829,12 +7792,6 @@ packages:resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==}- resolution: {integrity: sha512-6HVa5oIrgMC6aA6WF6XyyqbhRPJrKR02L20+2+zpDtO5QAzGHAUGw5TKQvwi5vctNnRHkJYmjAhRVQF2EKdTQw==}-- resolution: {integrity: sha512-csuQ9x3Yr0cEIs/Zgx/OEt9iBw9vqIunAPQkx19R/fiMq2oGVTgcMqO/V3Ybqefr1TBvosI6jU539ksaBULJyA==}-resolution: {integrity: sha512-YgBpdJHPyQ2UE5x+hlSXcnejzAvD0b22U2OuAP+8OnlJT+PjWPxtgmGqKKc+RgTM63U9gN0YzrYc71R2WT/hTA==}engines: {node: '>=18'}@@ -8259,15 +8216,6 @@ packages:resolution: {integrity: sha512-J8rn6v4DBb2nnFqkqwy6/NnTYMcgLA+sLr0iIO41qpv0n+ngb7ksag2tMRl0inb1bbO/esUwzW1vbJi7K0sI0g==}engines: {node: ^12.0.0 || ^14.0.0 || >=16.0.0}- resolution: {integrity: sha512-0YCaSCq2VRIebiaUviZNs0cBz1kg5kVS2UKUfNIx8YVs1cN3AV7NTctO5FOKBA+UT2BPJIWZauYHPqJODG50cg==}-- resolution: {integrity: sha512-8VhliFJAWRaUiVvREIiW2NXXTmHs4vMNnSzuJVhscgmGav3g9VDxLrQndI3dZZVVdp0ZO/5v0xmX516/7M9cng==}-- resolution: {integrity: sha512-6VwtthbV4o/7+OaAF9I5L5V3llLEsoPyq9P1JVXkedTP33c7MfCG0/5NOPcSJn0TzXcG9YUrR0gQSWioew3LDg==}-resolution: {integrity: sha512-sZuz1dYW/ZsfG17WSAG7eS85r5a0dDsvg+7BiiYR5o6lKCAtUrEwdmRmaGF6rwVj3LcmAeYkOWKEPlbPzN3Y3A==}engines: {node: ^12.0.0 || ^14.0.0 || >=16.0.0}@@ -8425,10 +8373,6 @@ packages:resolution: {integrity: sha512-PcByqNyT/38F2kDNi006HAMRJaULuBzq/FOsw3qdZvX/GA9W/jamDaRskgHjubHiftXK5sIFxLNkvrXUwcof6Q==}engines: {node: '>= 0.4'}- resolution: {integrity: sha512-OcRW72ngQvc+MSVWBRIovC/BTW589iUE/Vxav7VeM4RcY9MUjqB+IR6ILG1k9nuW6LiG/nct9uma4/MBtnZbmQ==}- engines: {node: '>=20'}-resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==}@@ -11255,6 +11199,8 @@ snapshots:'@oxlint/[email protected]': {}+ '@pierre/[email protected]': {}+'@pkgr/[email protected]': {}'@playwright/[email protected]':@@ -11407,46 +11353,6 @@ snapshots:eslint: 10.13.0([email protected])([email protected])oxc-parser: 0.148.0- '@shikijs/[email protected]':- dependencies:- '@shikijs/primitive': 4.5.0- '@shikijs/types': 4.5.0- '@shikijs/vscode-textmate': 10.0.2- '@types/hast': 3.0.5- hast-util-to-html: 9.0.5-- '@shikijs/[email protected]':- dependencies:- '@shikijs/types': 4.5.0- '@shikijs/vscode-textmate': 10.0.2- oniguruma-to-es: 4.3.6-- '@shikijs/[email protected]':- dependencies:- '@shikijs/types': 4.5.0- '@shikijs/vscode-textmate': 10.0.2-- '@shikijs/[email protected]':- dependencies:- '@shikijs/types': 4.5.0-- '@shikijs/[email protected]':- dependencies:- '@shikijs/types': 4.5.0- '@shikijs/vscode-textmate': 10.0.2- '@types/hast': 3.0.5-- '@shikijs/[email protected]':- dependencies:- '@shikijs/types': 4.5.0-- '@shikijs/[email protected]':- dependencies:- '@shikijs/vscode-textmate': 10.0.2- '@types/hast': 3.0.5-- '@shikijs/[email protected]': {}-'@shuding/[email protected]':dependencies:fflate: 0.7.5@@ -14034,20 +13940,6 @@ snapshots:'@ungap/structured-clone': 1.3.2unist-util-position: 5.0.0- dependencies:- '@types/hast': 3.0.5- '@types/unist': 3.0.3- ccount: 2.0.1- comma-separated-tokens: 2.0.3- hast-util-whitespace: 3.0.0- html-void-elements: 3.0.0- mdast-util-to-hast: 13.2.1- property-information: 7.2.0- space-separated-tokens: 2.0.2- stringify-entities: 4.0.4- zwitch: 2.0.4-dependencies:'@types/estree': 1.0.9@@ -15109,14 +15001,6 @@ snapshots:dependencies:wrappy: 1.0.2- [email protected]: {}-- dependencies:- oniguruma-parser: 0.12.2- regex: 6.1.0- regex-recursion: 6.0.2-dependencies:default-browser: 5.5.1@@ -15774,16 +15658,6 @@ snapshots:dependencies:'@eslint-community/regexpp': 4.12.2- dependencies:- regex-utilities: 2.3.0-- [email protected]: {}-- dependencies:- regex-utilities: 2.3.0-dependencies:'@eslint-community/regexpp': 4.12.2@@ -15997,17 +15871,6 @@ snapshots:- dependencies:- '@shikijs/core': 4.5.0- '@shikijs/engine-javascript': 4.5.0- '@shikijs/engine-oniguruma': 4.5.0- '@shikijs/langs': 4.5.0- '@shikijs/themes': 4.5.0- '@shikijs/types': 4.5.0- '@shikijs/vscode-textmate': 10.0.2- '@types/hast': 3.0.5-@@ -17024,6 +16887,7 @@ time:'@orpc/[email protected]': '2026-10-14T14:01:38.932Z''@orpc/[email protected]': '2026-10-14T14:02:09.442Z''@orpc/[email protected]': '2026-10-14T14:01:31.626Z'+ '@pierre/[email protected]': '2026-10-21T08:43:01.821Z''@playwright/[email protected]': '2026-10-14T11:58:13.696Z''@remixicon/[email protected]': '2026-01-29T10:53:18.993Z''@sentry/[email protected]': '2026-10-08T16:40:00.026Z'@@ -17119,7 +16983,6 @@ time:[email protected]: '2026-09-18T13:19:15.594Z'[email protected]: '2026-07-13T17:23:36.705Z'[email protected]: '2026-10-08T23:18:02.688Z'- [email protected]: '2025-03-05T11:30:29.166Z'[email protected]: '2026-10-04T03:53:48.117Z'[email protected]: '2025-03-30T15:40:10.885Z'[email protected]: '2025-02-14T01:43:48.709Z'@@ -17173,7 +17036,6 @@ time:[email protected]: '2026-09-09T17:17:30.281Z'[email protected]: '2022-09-03T01:07:26.139Z'[email protected]: '2026-09-27T13:46:24.509Z'- [email protected]: '2026-10-01T06:46:31.760Z'[email protected]: '2026-09-25T08:35:02.910Z'[email protected]: '2026-02-11T22:42:31.720Z'[email protected]: '2026-03-09T17:23:55.795Z'diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yamlindex 21c760525c1..744e1e32130 100644--- a/pnpm-workspace.yaml+++ b/pnpm-workspace.yaml@@ -100,6 +100,7 @@ catalog:'@orpc/contract': 1.15.7'@orpc/openapi-client': 1.15.7'@orpc/tanstack-query': 1.15.7+ '@pierre/highlights': 0.2.0'@playwright/test': 1.64.1'@remixicon/react': 4.9.0'@sentry/react': 10.76.2@@ -194,7 +195,6 @@ catalog:foxact: 0.3.12fuse.js: 7.5.0happy-dom: 20.14.6- hast-util-to-jsx-runtime: 2.3.6hono: 4.13.13html-entities: 2.6.0html-to-image: 1.11.13@@ -247,7 +247,6 @@ catalog:scheduler: 0.28.0server-only: 0.0.1sharp: 0.35.5- shiki: 4.5.0socket.io-client: 4.8.4sortablejs: 1.15.7std-semver: 1.0.8diff --git a/web/app/components/base/markdown-blocks/__tests__/code-block-loading.spec.tsx b/web/app/components/base/markdown-blocks/__tests__/code-block-loading.spec.tsxindex ff1645142c2..63d9606ae34 100644--- a/web/app/components/base/markdown-blocks/__tests__/code-block-loading.spec.tsx+++ b/web/app/components/base/markdown-blocks/__tests__/code-block-loading.spec.tsx@@ -23,9 +23,6 @@ const runtime = vi.hoisted(() => {}})-vi.mock('../shiki-highlight', () => ({- highlightCode: async ({ code }: { code: string }) => <pre>{code}</pre>,-}))vi.mock('echarts-for-react', () => {runtime.chartLoaded = truereturn {@@ -65,7 +62,11 @@ it('loads chart, music, and SVG runtimes only when their code-block language isconst { rerender, container } = render(<CodeBlock className="language-typescript">const answer = 42</CodeBlock>,)- await waitFor(() => expect(screen.getByText('const answer = 42')).toBeInTheDocument())+ await waitFor(() =>+ expect(container.querySelector('.code-line-numbers code')).toHaveTextContent(+ 'const answer = 42',+ ),+ )expect(runtime.svgLoaded).toBe(false)expect(runtime.chartLoaded).toBe(false)expect(runtime.musicLoaded).toBe(false)diff --git a/web/app/components/base/markdown-blocks/__tests__/code-block.spec.tsx b/web/app/components/base/markdown-blocks/__tests__/code-block.spec.tsxindex aba81e2f23a..b27803e7dc7 100644--- a/web/app/components/base/markdown-blocks/__tests__/code-block.spec.tsx+++ b/web/app/components/base/markdown-blocks/__tests__/code-block.spec.tsx@@ -2,10 +2,7 @@ import { act, render, screen, waitFor } from '@testing-library/react'import userEvent from '@testing-library/user-event'import { Theme } from '@/types/app'import { CodeBlock } from '../code-block'--const { mockHighlightCode } = vi.hoisted(() => ({- mockHighlightCode: vi.fn(),-}))+import { highlightCode } from '../code-highlight'type UseThemeReturn = {theme: Theme@@ -73,9 +70,9 @@ vi.mock('@/hooks/use-theme', () => ({default: () => mockUseTheme(),}))-vi.mock('../shiki-highlight', () => ({- highlightCode: mockHighlightCode,-}))+// Keep the real highlighter so tests cover its output; the spy only forces+// the failure fallback.+vi.mock('../code-highlight', { spy: true })vi.mock('../../svg-gallery', () => ({default: ({ content }: { content: string }) => <div data-testid="svg-renderer">{content}</div>,@@ -142,15 +139,17 @@ const findEchartsHost = async () => {return document.querySelector('.echarts-for-react') as HTMLDivElement}+const findHighlightedKeyword = async () => {+ await waitFor(() => {+ expect(document.querySelector('.code-line-numbers .line span'))!.toBeInTheDocument()+ })+ return document.querySelector('.code-line-numbers .line span') as HTMLSpanElement+}+describe('CodeBlock', () => {beforeEach(() => {vi.clearAllMocks()mockUseTheme.mockReturnValue({ theme: Theme.light })- mockHighlightCode.mockImplementation(async ({ code, language }) => (- <pre className="shiki">- <code className={`language-${language}`}>{code}</code>- </pre>- ))consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})consoleWarnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})clientWidthSpy = vi.spyOn(HTMLElement.prototype, 'clientWidth', 'get').mockReturnValue(900)@@ -218,11 +217,9 @@ describe('CodeBlock', () => {render(<CodeBlock className="language-javascript">const x = 1;</CodeBlock>)expect(screen.getByText('JavaScript'))!.toBeInTheDocument()- await waitFor(() => {- expect(document.querySelector('code.language-javascript')?.textContent).toContain(- 'const x = 1;',- )- })+ // github-light keyword color+ expect(await findHighlightedKeyword()).toHaveStyle({ color: '#d73a49' })+ expect(document.querySelector('.code-line-numbers code')?.textContent).toBe('const x = 1;')})it('should format unknown language labels with capitalized fallback when language is not in map', () => {@@ -257,22 +254,20 @@ describe('CodeBlock', () => {render(<CodeBlock className="language-javascript">const y = 2;</CodeBlock>)expect(screen.getByText('JavaScript'))!.toBeInTheDocument()- await waitFor(() => {- expect(document.querySelector('code.language-javascript')?.textContent).toContain(- 'const y = 2;',- )- })+ // github-dark keyword color+ expect(await findHighlightedKeyword()).toHaveStyle({ color: '#f97583' })})- it('should fall back to plain code block when shiki highlighting fails', async () => {- mockHighlightCode.mockRejectedValueOnce(new Error('highlight failed'))+ it('should fall back to plain code block when highlighting fails', async () => {+ vi.mocked(highlightCode).mockRejectedValueOnce(new Error('highlight failed'))render(<CodeBlock className="language-javascript">const z = 3;</CodeBlock>)await waitFor(() => {- expect(screen.getByText('const z = 3;'))!.toBeInTheDocument()+ expect(consoleErrorSpy).toHaveBeenCalledWith('Code highlighting failed:', expect.any(Error))})- expect(document.querySelector('code.language-javascript')).toBeNull()+ expect(screen.getByText('const z = 3;'))!.toBeInTheDocument()+ expect(document.querySelector('.code-line-numbers')).toBeNull()})})diff --git a/web/app/components/base/markdown-blocks/__tests__/shiki-highlight.spec.tsx b/web/app/components/base/markdown-blocks/__tests__/code-highlight.spec.tsxsimilarity index 52%rename from web/app/components/base/markdown-blocks/__tests__/shiki-highlight.spec.tsxrename to web/app/components/base/markdown-blocks/__tests__/code-highlight.spec.tsxindex 8e6a19e1ffa..a3d34738c53 100644--- a/web/app/components/base/markdown-blocks/__tests__/shiki-highlight.spec.tsx+++ b/web/app/components/base/markdown-blocks/__tests__/code-highlight.spec.tsx@@ -1,42 +1,16 @@-import type { createSingletonShorthands } from 'shiki/core'-import type * as shikiCore from 'shiki/core'+import { codeToTokens } from '@pierre/highlights'import { renderToStaticMarkup } from 'react-dom/server'-import { highlightCode } from '../shiki-highlight'--// `shiki/core` only exposes `getSingletonHighlighter` as a property of the-// shorthand bundle returned by `createSingletonShorthands`, not as a top-level-// symbol. Derive the accessor signature from the return type so wrapped calls-// remain correctly typed without importing a non-exported name.-type SingletonAccessor = ReturnType<typeof createSingletonShorthands>['getSingletonHighlighter']--// Mock the singleton accessor so tests can assert that the WASM engine is not-// loaded for plain-text fences while still letting supported-language tests-// observe a real `getSingletonHighlighter` invocation.-const { mockGetSingletonHighlighter } = vi.hoisted(() => ({- mockGetSingletonHighlighter: vi.fn(),-}))--vi.mock('shiki/core', async (importOriginal) => {- const actual = await importOriginal<typeof shikiCore>()- const realCreateSingletonShorthands = actual.createSingletonShorthands- return {- ...actual,- createSingletonShorthands: (factory: Parameters<typeof realCreateSingletonShorthands>[0]) => {- const real = realCreateSingletonShorthands(factory)- // Wrap real `getSingletonHighlighter` so tests can observe invocations.- const realGet = real.getSingletonHighlighter- const wrapped: SingletonAccessor = (...args: Parameters<typeof realGet>) => {- mockGetSingletonHighlighter(...args)- return realGet(...args)- }- return { getSingletonHighlighter: wrapped }- },- }-})+import { highlightCode } from '../code-highlight'++// Spy on the real tokenizer so tests can assert that plain-text and unsupported+// fences never reach the WebAssembly lexers.+vi.mock('@pierre/highlights', { spy: true })++const mockCodeToTokens = vi.mocked(codeToTokens)describe('README code highlighting', () => {beforeEach(() => {- mockGetSingletonHighlighter.mockReset()+ mockCodeToTokens.mockClear()})it.each(['github-light', 'github-dark'] as const)('highlights dotenv with %s', async (theme) => {@@ -47,7 +21,8 @@ describe('README code highlighting', () => {expect(result).toContain('your-api-key')expect(result).toContain('OPENAI_ORGANIZATION=org-id')expect(result).toContain('<span style="color:')- expect(mockGetSingletonHighlighter).toHaveBeenCalledTimes(1)+ expect(mockCodeToTokens).toHaveBeenCalledTimes(1)+ expect(mockCodeToTokens.mock.calls[0]![1]).toMatchObject({ lang: 'bash' })})it('renders unsupported languages as readable, escaped plain text', async () => {@@ -61,11 +36,11 @@ describe('README code highlighting', () => {)expect(result).toContain('<custom>example</custom>')- expect(mockGetSingletonHighlighter).not.toHaveBeenCalled()+ expect(mockCodeToTokens).not.toHaveBeenCalled()})- it.each(['', ' '])(- 'renders a blank language %j without loading the WASM engine',+ it.each(['', ' ', 'text', 'TXT'])(+ 'renders the plain language %j without tokenizing',async (language) => {const code = 'plain line one\nplain line two'const result = renderToStaticMarkup(@@ -74,7 +49,7 @@ describe('README code highlighting', () => {expect(result).toContain('plain line one')expect(result).toContain('plain line two')- expect(mockGetSingletonHighlighter).not.toHaveBeenCalled()+ expect(mockCodeToTokens).not.toHaveBeenCalled()},)@@ -92,8 +67,26 @@ describe('README code highlighting', () => {const document = new DOMParser().parseFromString(result, 'text/html')expect(document.querySelector('code')?.textContent).toBe(lines.join('\n'))+ expect(document.querySelectorAll('.line')).toHaveLength(lines.length)expect(document.querySelector('custom')).toBeNull()- expect(mockGetSingletonHighlighter).not.toHaveBeenCalled()+ },+ )++ it.each(['\n', '\r\n'])(+ 'renders one line element per highlighted line with %j line endings',+ async (lineEnding) => {+ const lines = ['const a = 1', '', ' const b = a']+ const result = renderToStaticMarkup(+ await highlightCode({+ code: lines.join(lineEnding),+ language: 'ts',+ theme: 'github-dark',+ }),+ )+ const document = new DOMParser().parseFromString(result, 'text/html')++ expect(document.querySelector('code')?.textContent).toBe(lines.join('\n'))+ expect(document.querySelectorAll('.line')).toHaveLength(lines.length)},)@@ -104,20 +97,33 @@ describe('README code highlighting', () => {)expect(result).toContain('a < b && b > c & "x"')- expect(mockGetSingletonHighlighter).not.toHaveBeenCalled()})- it('preserves highlighting for bundled language aliases', async () => {+ it('escapes HTML-sensitive characters in highlighted fences', async () => {+ const result = renderToStaticMarkup(+ await highlightCode({+ code: '<div class="x">&</div>',+ language: 'html',+ theme: 'github-light',+ }),+ )++ expect(result).not.toContain('<div class="x">')+ expect(result).toContain('<')+ expect(result).toContain('&amp;')+ })++ it.each(['js', 'Python', 'go'])('highlights the bundled language %s', async (language) => {const result = renderToStaticMarkup(await highlightCode({code: 'const count = 1',- language: 'js',+ language,theme: 'github-light',}),)- expect(result).toContain('const')+ expect(result).toContain('count')expect(result).toContain('<span style="color:')- expect(mockGetSingletonHighlighter).toHaveBeenCalledTimes(1)+ expect(mockCodeToTokens).toHaveBeenCalledTimes(1)})})diff --git a/web/app/components/base/markdown-blocks/code-block.tsx b/web/app/components/base/markdown-blocks/code-block.tsxindex 22faea4b0b4..2bfa44c6a8e 100644--- a/web/app/components/base/markdown-blocks/code-block.tsx+++ b/web/app/components/base/markdown-blocks/code-block.tsx@@ -2,7 +2,7 @@ import type { EChartsOption, EChartsType } from 'echarts'import type EChartsReact from 'echarts-for-react'import type { EChartsReactProps } from 'echarts-for-react'import type { JSX } from 'react'-import type { CodeTheme } from './shiki-highlight'+import type { CodeTheme } from './code-highlight'import { IconButton } from '@langgenius/dify-ui/icon-button'import { Toggle } from '@langgenius/dify-ui/toggle'import dynamic from 'next/dynamic'@@ -21,7 +21,7 @@ import CopyIcon from '@/app/components/base/copy-icon'import ErrorBoundary from '@/app/components/base/markdown/error-boundary'import useTheme from '@/hooks/use-theme'import { Theme } from '@/types/app'-import { highlightCode } from './shiki-highlight'+import { highlightCode } from './code-highlight'const MermaidDiagram = dynamic(() => import('./mermaid-diagram').then((module) => module.MermaidDiagram),@@ -76,7 +76,7 @@ const getCorrectCapitalizationLanguageName = (language: string) => {// visit https://reactjs.org/docs/error-decoder.html?invariant=185 for the full message// or use the non-minified dev environment for full errors and additional helpful warnings.-const ShikiCodeBlock = memo(+const HighlightedCodeBlock = memo(({code,language,@@ -102,7 +102,7 @@ const ShikiCodeBlock = memo(if (!cancelled) setNodes(result)}).catch((error) => {- console.error('Shiki highlighting failed:', error)+ console.error('Code highlighting failed:', error)if (!cancelled) setNodes(undefined)})@@ -135,14 +135,14 @@ const ShikiCodeBlock = memo(borderBottomRightRadius: '10px',overflow: 'auto',}}- className="shiki-line-numbers [&_pre]:m-0! [&_pre]:rounded-t-none! [&_pre]:rounded-b-[10px]! [&_pre]:bg-components-input-bg-normal! [&_pre]:py-2!"+ className="code-line-numbers [&_pre]:m-0! [&_pre]:rounded-t-none! [&_pre]:rounded-b-[10px]! [&_pre]:bg-components-input-bg-normal! [&_pre]:py-2!">{nodes}</div>)},)-ShikiCodeBlock.displayName = 'ShikiCodeBlock'+HighlightedCodeBlock.displayName = 'HighlightedCodeBlock'// Define ECharts event parameter typestype EChartsEventParams = {@@ -525,7 +525,7 @@ export const CodeBlock: any = memo(({ inline, className, children = '', ...props)default:return (- <ShikiCodeBlock+ <HighlightedCodeBlockcode={content}language={match?.[1] || 'text'}theme={isDarkMode ? 'github-dark' : 'github-light'}diff --git a/web/app/components/base/markdown-blocks/code-highlight.tsx b/web/app/components/base/markdown-blocks/code-highlight.tsxnew file mode 100644index 00000000000..d28aae58bc3--- /dev/null+++ b/web/app/components/base/markdown-blocks/code-highlight.tsx@@ -0,0 +1,86 @@+import type { ThemedToken } from '@pierre/highlights'+import type { CSSProperties, JSX } from 'react'+import { Fragment } from 'react'++const themes = {+ 'github-dark': () => import('@pierre/highlights/themes/github-dark'),+ 'github-light': () => import('@pierre/highlights/themes/github-light'),+}++export type CodeTheme = keyof typeof themes++// Highlights bundles a fixed lexer set. Map README fence names it lacks to a+// close lexer; other unsupported names render as plain text.+const languageAliases: Record<string, string> = {+ dotenv: 'bash',+}++const plainLanguages = new Set(['', 'plain', 'plaintext', 'text', 'txt'])++type HighlightCodeOptions = {+ code: string+ language: string+ theme: CodeTheme+}++type CodeLine = Pick<ThemedToken, 'content' | 'color' | 'fontStyle'>[]++const getTokenStyle = ({ color, fontStyle = 0 }: CodeLine[number]): CSSProperties | undefined => {+ if (!color && !fontStyle) return undefined+ return {+ color,+ // Highlights only emits Shiki's italic (1) and bold (2) flags.+ fontStyle: fontStyle & 1 ? 'italic' : undefined,+ fontWeight: fontStyle & 2 ? 'bold' : undefined,+ }+}++const renderLines = (lines: CodeLine[], style?: CSSProperties) => (+ <pre style={style}>+ <code>+ {lines.map((tokens, lineIndex) => (+ // oxlint-disable-next-line react/no-array-index-key -- Lines are positional and re-render together.+ <Fragment key={lineIndex}>+ {lineIndex > 0 && '\n'}+ {/* The code-block gutter uses .line::before for its line numbers. */}+ <span className="line">+ {tokens.map((token, tokenIndex) => (+ // oxlint-disable-next-line react/no-array-index-key -- Tokens are positional within their line.+ <span key={tokenIndex} style={getTokenStyle(token)}>+ {token.content}+ </span>+ ))}+ </span>+ </Fragment>+ ))}+ </code>+ </pre>+)++const renderPlainText = (code: string) =>+ renderLines(code.split(/\r?\n/).map((content) => [{ content }]))++export const highlightCode = async ({+ code,+ language,+ theme,+}: HighlightCodeOptions): Promise<JSX.Element> => {+ const normalizedLanguage = language.trim().toLowerCase()+ const lang = languageAliases[normalizedLanguage] ?? normalizedLanguage++ // Plain text does not need syntax highlighting, so skip loading the+ // WebAssembly lexers for it (see #42943).+ if (plainLanguages.has(lang)) return renderPlainText(code)++ const [{ codeToTokens, isSupportedLanguage }, { default: themeDefinition }] = await Promise.all([+ import('@pierre/highlights'),+ themes[theme](),+ ])++ // README fences may name languages outside the bundled lexers. Keep them+ // readable instead of letting Highlights throw a RangeError.+ if (!isSupportedLanguage(lang)) return renderPlainText(code)++ const { tokens, fg, bg } = codeToTokens(code, { lang, theme: themeDefinition })+ return renderLines(tokens, { backgroundColor: bg, color: fg })+}diff --git a/web/app/components/base/markdown-blocks/shiki-highlight.tsx b/web/app/components/base/markdown-blocks/shiki-highlight.tsxdeleted file mode 100644index b3085dbd81b..00000000000--- a/web/app/components/base/markdown-blocks/shiki-highlight.tsx+++ /dev/null@@ -1,97 +0,0 @@-import type { JSX } from 'react'-import { toJsxRuntime } from 'hast-util-to-jsx-runtime'-import { Fragment } from 'react'-import { jsx, jsxs } from 'react/jsx-runtime'-import { createBundledHighlighter, createSingletonShorthands } from 'shiki/core'-import { createOnigurumaEngine } from 'shiki/engine/oniguruma'-import { codeLanguages } from './shiki-languages'--const createHighlighter = createBundledHighlighter({- langs: codeLanguages,- themes: {- 'github-dark': () => import('shiki/themes/github-dark.mjs'),- 'github-light': () => import('shiki/themes/github-light.mjs'),- },- engine: () => createOnigurumaEngine(import('shiki/wasm')),-})--const { getSingletonHighlighter } = createSingletonShorthands(createHighlighter)--export type CodeTheme = 'github-dark' | 'github-light'--type HighlightCodeOptions = {- code: string- language: string- theme: CodeTheme-}--export const highlightCode = async ({- code,- language,- theme,-}: HighlightCodeOptions): Promise<JSX.Element> => {- const normalizedLanguage = language.trim().toLowerCase()- const lang =- normalizedLanguage === 'dotenv' || Object.hasOwn(codeLanguages, normalizedLanguage)- ? normalizedLanguage- : 'text'-- // Plain text and unsupported languages do not need syntax highlighting, so- // skip `getSingletonHighlighter` to avoid loading the WASM engine for them- // (see #42943). The shiki engine is still loaded on the first real highlight- // call, so subsequent supported-language code blocks render normally.- if (lang === 'text') {- // Hand-built minimal HAST tree; typed via `toJsxRuntime`'s own parameter- // so we don't need a top-level `hast` import (the shipped `@types/hast`- // transitively pulls in `hast` module declarations that we cannot resolve).- const plainHast: Parameters<typeof toJsxRuntime>[0] = {- type: 'root',- children: [- {- type: 'element',- tagName: 'pre',- properties: {},- children: [- {- type: 'element',- tagName: 'code',- properties: {},- // The code-block gutter uses .line::before for its line numbers.- children: code.split(/\r?\n/).flatMap((line, index) => [- { type: 'text' as const, value: index === 0 ? '' : '\n' },- {- type: 'element' as const,- tagName: 'span',- properties: { className: ['line'] },- children: [{ type: 'text' as const, value: line }],- },- ]),- },- ],- },- ],- }- return toJsxRuntime(plainHast, {- Fragment,- jsx,- jsxs,- }) as JSX.Element- }-- // README fences may name languages outside the web bundle. Load dotenv on- // demand and keep unknown languages readable without throwing an error.- const highlighter = await getSingletonHighlighter({- langs: lang === 'dotenv' ? [(await import('shiki/langs/dotenv.mjs')).default] : [lang],- themes: [theme],- })- const hast = highlighter.codeToHast(code, {- lang,- theme,- })-- return toJsxRuntime(hast, {- Fragment,- jsx,- jsxs,- }) as JSX.Element-}diff --git a/web/app/components/base/markdown-blocks/shiki-languages.ts b/web/app/components/base/markdown-blocks/shiki-languages.tsdeleted file mode 100644index 5e2b8cb64df..00000000000--- a/web/app/components/base/markdown-blocks/shiki-languages.ts+++ /dev/null@@ -1,82 +0,0 @@-// Keep the existing Shiki web language set and aliases without importing its all-theme bundle.-export const codeLanguages = {- 'angular-html': () => import('shiki/langs/angular-html.mjs'),- 'angular-ts': () => import('shiki/langs/angular-ts.mjs'),- astro: () => import('shiki/langs/astro.mjs'),- blade: () => import('shiki/langs/blade.mjs'),- c: () => import('shiki/langs/c.mjs'),- coffee: () => import('shiki/langs/coffee.mjs'),- coffeescript: () => import('shiki/langs/coffee.mjs'),- cpp: () => import('shiki/langs/cpp.mjs'),- 'c++': () => import('shiki/langs/cpp.mjs'),- css: () => import('shiki/langs/css.mjs'),- csv: () => import('shiki/langs/csv.mjs'),- glsl: () => import('shiki/langs/glsl.mjs'),- graphql: () => import('shiki/langs/graphql.mjs'),- gql: () => import('shiki/langs/graphql.mjs'),- haml: () => import('shiki/langs/haml.mjs'),- handlebars: () => import('shiki/langs/handlebars.mjs'),- hbs: () => import('shiki/langs/handlebars.mjs'),- html: () => import('shiki/langs/html.mjs'),- 'html-derivative': () => import('shiki/langs/html-derivative.mjs'),- http: () => import('shiki/langs/http.mjs'),- hurl: () => import('shiki/langs/hurl.mjs'),- imba: () => import('shiki/langs/imba.mjs'),- java: () => import('shiki/langs/java.mjs'),- javascript: () => import('shiki/langs/javascript.mjs'),- js: () => import('shiki/langs/javascript.mjs'),- cjs: () => import('shiki/langs/javascript.mjs'),- mjs: () => import('shiki/langs/javascript.mjs'),- jinja: () => import('shiki/langs/jinja.mjs'),- jison: () => import('shiki/langs/jison.mjs'),- json: () => import('shiki/langs/json.mjs'),- json5: () => import('shiki/langs/json5.mjs'),- jsonc: () => import('shiki/langs/jsonc.mjs'),- jsonl: () => import('shiki/langs/jsonl.mjs'),- jsx: () => import('shiki/langs/jsx.mjs'),- julia: () => import('shiki/langs/julia.mjs'),- jl: () => import('shiki/langs/julia.mjs'),- less: () => import('shiki/langs/less.mjs'),- markdown: () => import('shiki/langs/markdown.mjs'),- md: () => import('shiki/langs/markdown.mjs'),- marko: () => import('shiki/langs/marko.mjs'),- mdc: () => import('shiki/langs/mdc.mjs'),- mdx: () => import('shiki/langs/mdx.mjs'),- php: () => import('shiki/langs/php.mjs'),- postcss: () => import('shiki/langs/postcss.mjs'),- pug: () => import('shiki/langs/pug.mjs'),- jade: () => import('shiki/langs/pug.mjs'),- python: () => import('shiki/langs/python.mjs'),- py: () => import('shiki/langs/python.mjs'),- r: () => import('shiki/langs/r.mjs'),- regexp: () => import('shiki/langs/regexp.mjs'),- regex: () => import('shiki/langs/regexp.mjs'),- sass: () => import('shiki/langs/sass.mjs'),- scss: () => import('shiki/langs/scss.mjs'),- shellscript: () => import('shiki/langs/shellscript.mjs'),- bash: () => import('shiki/langs/shellscript.mjs'),- sh: () => import('shiki/langs/shellscript.mjs'),- shell: () => import('shiki/langs/shellscript.mjs'),- zsh: () => import('shiki/langs/shellscript.mjs'),- smithy: () => import('shiki/langs/smithy.mjs'),- sql: () => import('shiki/langs/sql.mjs'),- stylus: () => import('shiki/langs/stylus.mjs'),- styl: () => import('shiki/langs/stylus.mjs'),- svelte: () => import('shiki/langs/svelte.mjs'),- 'ts-tags': () => import('shiki/langs/ts-tags.mjs'),- lit: () => import('shiki/langs/ts-tags.mjs'),- tsx: () => import('shiki/langs/tsx.mjs'),- typescript: () => import('shiki/langs/typescript.mjs'),- ts: () => import('shiki/langs/typescript.mjs'),- cts: () => import('shiki/langs/typescript.mjs'),- mts: () => import('shiki/langs/typescript.mjs'),- vue: () => import('shiki/langs/vue.mjs'),- 'vue-html': () => import('shiki/langs/vue-html.mjs'),- 'vue-vine': () => import('shiki/langs/vue-vine.mjs'),- wasm: () => import('shiki/langs/wasm.mjs'),- wgsl: () => import('shiki/langs/wgsl.mjs'),- wit: () => import('shiki/langs/wit.mjs'),- xml: () => import('shiki/langs/xml.mjs'),- yaml: () => import('shiki/langs/yaml.mjs'),- yml: () => import('shiki/langs/yaml.mjs'),-}diff --git a/web/app/components/datasets/documents/detail/completed/segment-card/__tests__/index.spec.tsx b/web/app/components/datasets/documents/detail/completed/segment-card/__tests__/index.spec.tsxindex 38325797cfb..d5854243ff2 100644--- a/web/app/components/datasets/documents/detail/completed/segment-card/__tests__/index.spec.tsx+++ b/web/app/components/datasets/documents/detail/completed/segment-card/__tests__/index.spec.tsx@@ -87,7 +87,7 @@ vi.mock('@/app/components/datasets/common/image-list', () => ({),}))-// Markdown uses next/dynamic and shiki (ESM)+// Markdown loads its code blocks through next/dynamicvi.mock('@/app/components/base/markdown', () => ({Markdown: ({ content, className }: { content: string; className?: string }) => (<div data-testid="markdown" className={`markdown-body ${className || ''}`}>diff --git a/web/app/styles/tailwind-core.css b/web/app/styles/tailwind-core.cssindex fe791364be9..50a162384f2 100644--- a/web/app/styles/tailwind-core.css+++ b/web/app/styles/tailwind-core.css@@ -150,11 +150,11 @@background-clip: text;}- /* Shiki code block line numbers */- .shiki-line-numbers code {+ /* Highlighted code block line numbers */+ .code-line-numbers code {counter-reset: line;}- .shiki-line-numbers .line::before {+ .code-line-numbers .line::before {counter-increment: line;content: counter(line);display: inline-block;diff --git a/web/package.json b/web/package.jsonindex fc7d675aa3d..4992a9f2846 100644--- a/web/package.json+++ b/web/package.json@@ -59,6 +59,7 @@"@orpc/contract": "catalog:","@orpc/openapi-client": "catalog:","@orpc/tanstack-query": "catalog:",+ "@pierre/highlights": "catalog:","@remixicon/react": "catalog:","@sentry/react": "catalog:","@streamdown/math": "catalog:",@@ -89,7 +90,6 @@"fast-deep-equal": "catalog:","foxact": "catalog:","fuse.js": "catalog:",- "hast-util-to-jsx-runtime": "catalog:","html-entities": "catalog:","html-to-image": "catalog:","i18next": "catalog:",@@ -128,7 +128,6 @@"remark-directive": "catalog:","scheduler": "catalog:","server-only": "catalog:",- "shiki": "catalog:","socket.io-client": "catalog:","sortablejs": "catalog:","std-semver": "catalog:",