Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

refactor(web): highlight markdown code blocks with @pierre/highlights (#44522)

Stephen Zhou07a1debf7cea
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 3ef00da27a2..2fabee50a6e 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -260,6 +260,9 @@ catalogs:
'@orpc/tanstack-query':
specifier: 1.15.7
version: 1.15.7
+ '@pierre/highlights':
+ specifier: 0.2.0
+ version: 0.2.0
'@playwright/test':
specifier: 1.64.1
version: 1.64.1
@@ -536,9 +539,6 @@ catalogs:
fuse.js:
specifier: 7.5.0
version: 7.5.0
- hast-util-to-jsx-runtime:
- specifier: 2.3.6
- version: 2.3.6
hono:
specifier: 4.13.13
version: 4.13.13
@@ -695,9 +695,6 @@ catalogs:
sharp:
specifier: 0.35.5
version: 0.35.5
- shiki:
- specifier: 4.5.0
- version: 4.5.0
socket.io-client:
specifier: 4.8.4
version: 4.8.4
@@ -1465,6 +1462,9 @@ importers:
'@orpc/tanstack-query':
specifier: 'catalog:'
version: 1.15.7(@orpc/[email protected])(@tanstack/[email protected])
+ '@pierre/highlights':
+ specifier: 'catalog:'
+ version: 0.2.0
'@remixicon/react':
specifier: 'catalog:'
version: 4.9.0([email protected])
@@ -1555,9 +1555,6 @@ importers:
fuse.js:
specifier: 'catalog:'
version: 7.5.0
- hast-util-to-jsx-runtime:
- specifier: 'catalog:'
- version: 2.3.6([email protected])
html-entities:
specifier: 'catalog:'
version: 2.6.0
@@ -1672,9 +1669,6 @@ importers:
server-only:
specifier: 'catalog:'
version: 0.0.1
- shiki:
- specifier: 'catalog:'
- version: 4.5.0
socket.io-client:
specifier: 'catalog:'
version: 4.8.4([email protected])
@@ -4114,6 +4108,9 @@ packages:
resolution: {integrity: sha512-G2xbfWTrbqYRhR/vfeKNMQf/sS3/ti0iKJAAkRxHuXWZH/hne/dyIb9QKk5LuR6cJLARxsL6FMoAYmfspd74MQ==}
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
+ '@pierre/[email protected]':
+ resolution: {integrity: sha512-hTpcOU1EzDAP4bYtEhxmMu2jwMvtkUydRUBUpWMSHTs3dw0IaC3Ld5ansMy18L0GE3b6Vfl2OGkLK+pWWpnBmw==}
+
resolution: {integrity: sha512-SEeaJLb3qBNF/OaXnaR1NmmBbFYk1zC0ZH/52fATcRPLFg/p791YrcyFFy44Bo9sLaGuSuLp5Q6axbb/O+v/RA==}
engines: {node: ^14.18.0 || >=16.0.0}
@@ -4232,37 +4229,6 @@ packages:
eslint:
optional: true
- '@shikijs/[email protected]':
- resolution: {integrity: sha512-6CIgwq+bt2JZ3OSjb/nBkkwEf/CfpHcbam+Mg5Pllco7HAGRQ2/EFhZrrJtbyNTgKwknfVS+llsXGygiDGwJWw==}
- engines: {node: '>=20'}
-
- '@shikijs/[email protected]':
- resolution: {integrity: sha512-Ed6UdEj84LyYkwnxM5CvSMWAD+id9ExTYri0/+x41U9zrgNifPSS9lZm0JqBP0VfYI8yr8tY5nbNq9+p9R0kFA==}
- engines: {node: '>=20'}
-
- '@shikijs/[email protected]':
- resolution: {integrity: sha512-dKVmbJecVdB9recl8a/CU4jdpNXyjJQrIcesIkpS70s5HS02ohpnCSpMUbotPnEUzbAcLQ+S4hwZYbkBBiTcJw==}
- engines: {node: '>=20'}
-
- '@shikijs/[email protected]':
- resolution: {integrity: sha512-wwBTqwduXSz0X4v9Qa2EWeFaYJEw1clCncWjoRNQRbVDu3SPRTRWucnouYapKj3b8Q+sk/41Vbc4B5c8nuYzLA==}
- engines: {node: '>=20'}
-
- '@shikijs/[email protected]':
- resolution: {integrity: sha512-UM7ShlA8oz0OWxLLO8BmO9Ip6fVqdkt8yF0EH7nOCqHfrx1eEAzF0ZUVK7L+I8NEgeu1PEWSRlxaiPPeCb4iDg==}
- engines: {node: '>=20'}
-
- '@shikijs/[email protected]':
- resolution: {integrity: sha512-zGb87H3UBbpuI4yoh5UguB1XbU+2BDU3kflM7UfZ+Cy2hWo9lBtqjSVK20gmLY6ukYl040hAypGXMgud9vul8Q==}
- engines: {node: '>=20'}
-
- '@shikijs/[email protected]':
- resolution: {integrity: sha512-drtPrtD8N0DENyD2cyt6Ajy+IWsiCLgMxODwLTpvH1UsN6a9jf90tNTPjqwHvA+0Lzu9e0txk8DjfqH1p9UJRw==}
- engines: {node: '>=20'}
-
- '@shikijs/[email protected]':
- resolution: {integrity: sha512-83yeghZ2xxin3Nj8z1NMd/NCuca+gsYXswywDy5bHvwlWL8tpTQmzGeUuHd9FC3E/SBEMvzJRwWEOz5gGes9Qg==}
-
'@shuding/[email protected]':
resolution: {integrity: sha512-3NgmNyH3l/Hv6EvsWJbsvpcpUba6R8IREQ83nH83cyakCw7uM1arZKNfHwv1Wz6jgqrF/j4x5ELvR6PnK9nTcA==}
engines: {node: '>= 8.0.0'}
@@ -6790,9 +6756,6 @@ packages:
resolution: {integrity: sha512-3yTWghByc50aGS7JlGhk61SPenfE/p1oaFeNwkOOyrscaOkMGrcW9+Cy/QAIOBpZxP1yqDIzFMR0+Np0i0+usg==}
- resolution: {integrity: sha512-OguPdidb+fbHQSU4Q4ZiLKnzWo8Wwsf5bZfbvu7//a9oTYoqD/fWpe96NuHkoS9h0ccGOTe0C4NGXdtS0iObOw==}
-
resolution: {integrity: sha512-zl6s8LwNyo1P9uw+XJGvZtdFF1GdAkOg8ujOw+4Pyb76874fLps4ueHXDhXWdk6YHQ6OgUtinliG7RsYvCbbBg==}
@@ -7829,12 +7792,6 @@ packages:
resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==}
- resolution: {integrity: sha512-6HVa5oIrgMC6aA6WF6XyyqbhRPJrKR02L20+2+zpDtO5QAzGHAUGw5TKQvwi5vctNnRHkJYmjAhRVQF2EKdTQw==}
-
- resolution: {integrity: sha512-csuQ9x3Yr0cEIs/Zgx/OEt9iBw9vqIunAPQkx19R/fiMq2oGVTgcMqO/V3Ybqefr1TBvosI6jU539ksaBULJyA==}
-
resolution: {integrity: sha512-YgBpdJHPyQ2UE5x+hlSXcnejzAvD0b22U2OuAP+8OnlJT+PjWPxtgmGqKKc+RgTM63U9gN0YzrYc71R2WT/hTA==}
engines: {node: '>=18'}
@@ -8259,15 +8216,6 @@ packages:
resolution: {integrity: sha512-J8rn6v4DBb2nnFqkqwy6/NnTYMcgLA+sLr0iIO41qpv0n+ngb7ksag2tMRl0inb1bbO/esUwzW1vbJi7K0sI0g==}
engines: {node: ^12.0.0 || ^14.0.0 || >=16.0.0}
- resolution: {integrity: sha512-0YCaSCq2VRIebiaUviZNs0cBz1kg5kVS2UKUfNIx8YVs1cN3AV7NTctO5FOKBA+UT2BPJIWZauYHPqJODG50cg==}
-
- resolution: {integrity: sha512-8VhliFJAWRaUiVvREIiW2NXXTmHs4vMNnSzuJVhscgmGav3g9VDxLrQndI3dZZVVdp0ZO/5v0xmX516/7M9cng==}
-
- resolution: {integrity: sha512-6VwtthbV4o/7+OaAF9I5L5V3llLEsoPyq9P1JVXkedTP33c7MfCG0/5NOPcSJn0TzXcG9YUrR0gQSWioew3LDg==}
-
resolution: {integrity: sha512-sZuz1dYW/ZsfG17WSAG7eS85r5a0dDsvg+7BiiYR5o6lKCAtUrEwdmRmaGF6rwVj3LcmAeYkOWKEPlbPzN3Y3A==}
engines: {node: ^12.0.0 || ^14.0.0 || >=16.0.0}
@@ -8425,10 +8373,6 @@ packages:
resolution: {integrity: sha512-PcByqNyT/38F2kDNi006HAMRJaULuBzq/FOsw3qdZvX/GA9W/jamDaRskgHjubHiftXK5sIFxLNkvrXUwcof6Q==}
engines: {node: '>= 0.4'}
- resolution: {integrity: sha512-OcRW72ngQvc+MSVWBRIovC/BTW589iUE/Vxav7VeM4RcY9MUjqB+IR6ILG1k9nuW6LiG/nct9uma4/MBtnZbmQ==}
- engines: {node: '>=20'}
-
resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==}
@@ -11255,6 +11199,8 @@ snapshots:
'@oxlint/[email protected]': {}
+ '@pierre/[email protected]': {}
+
'@pkgr/[email protected]': {}
'@playwright/[email protected]':
@@ -11407,46 +11353,6 @@ snapshots:
oxc-parser: 0.148.0
- '@shikijs/[email protected]':
- dependencies:
- '@shikijs/primitive': 4.5.0
- '@shikijs/types': 4.5.0
- '@shikijs/vscode-textmate': 10.0.2
- '@types/hast': 3.0.5
- hast-util-to-html: 9.0.5
-
- '@shikijs/[email protected]':
- dependencies:
- '@shikijs/types': 4.5.0
- '@shikijs/vscode-textmate': 10.0.2
- oniguruma-to-es: 4.3.6
-
- '@shikijs/[email protected]':
- dependencies:
- '@shikijs/types': 4.5.0
- '@shikijs/vscode-textmate': 10.0.2
-
- '@shikijs/[email protected]':
- dependencies:
- '@shikijs/types': 4.5.0
-
- '@shikijs/[email protected]':
- dependencies:
- '@shikijs/types': 4.5.0
- '@shikijs/vscode-textmate': 10.0.2
- '@types/hast': 3.0.5
-
- '@shikijs/[email protected]':
- dependencies:
- '@shikijs/types': 4.5.0
-
- '@shikijs/[email protected]':
- dependencies:
- '@shikijs/vscode-textmate': 10.0.2
- '@types/hast': 3.0.5
-
- '@shikijs/[email protected]': {}
-
'@shuding/[email protected]':
dependencies:
fflate: 0.7.5
@@ -14034,20 +13940,6 @@ snapshots:
'@ungap/structured-clone': 1.3.2
unist-util-position: 5.0.0
- dependencies:
- '@types/hast': 3.0.5
- '@types/unist': 3.0.3
- ccount: 2.0.1
- comma-separated-tokens: 2.0.3
- hast-util-whitespace: 3.0.0
- html-void-elements: 3.0.0
- mdast-util-to-hast: 13.2.1
- property-information: 7.2.0
- space-separated-tokens: 2.0.2
- stringify-entities: 4.0.4
- zwitch: 2.0.4
-
dependencies:
'@types/estree': 1.0.9
@@ -15109,14 +15001,6 @@ snapshots:
dependencies:
wrappy: 1.0.2
-
- dependencies:
- oniguruma-parser: 0.12.2
- regex: 6.1.0
- regex-recursion: 6.0.2
-
dependencies:
default-browser: 5.5.1
@@ -15774,16 +15658,6 @@ snapshots:
dependencies:
'@eslint-community/regexpp': 4.12.2
- dependencies:
- regex-utilities: 2.3.0
-
-
- dependencies:
- regex-utilities: 2.3.0
-
dependencies:
'@eslint-community/regexpp': 4.12.2
@@ -15997,17 +15871,6 @@ snapshots:
- dependencies:
- '@shikijs/core': 4.5.0
- '@shikijs/engine-javascript': 4.5.0
- '@shikijs/engine-oniguruma': 4.5.0
- '@shikijs/langs': 4.5.0
- '@shikijs/themes': 4.5.0
- '@shikijs/types': 4.5.0
- '@shikijs/vscode-textmate': 10.0.2
- '@types/hast': 3.0.5
-
@@ -17024,6 +16887,7 @@ time:
'@orpc/[email protected]': '2026-10-14T14:01:38.932Z'
'@orpc/[email protected]': '2026-10-14T14:02:09.442Z'
'@orpc/[email protected]': '2026-10-14T14:01:31.626Z'
+ '@pierre/[email protected]': '2026-10-21T08:43:01.821Z'
'@playwright/[email protected]': '2026-10-14T11:58:13.696Z'
'@remixicon/[email protected]': '2026-01-29T10:53:18.993Z'
'@sentry/[email protected]': '2026-10-08T16:40:00.026Z'
@@ -17119,7 +16983,6 @@ time:
[email protected]: '2026-09-18T13:19:15.594Z'
[email protected]: '2026-07-13T17:23:36.705Z'
[email protected]: '2026-10-08T23:18:02.688Z'
- [email protected]: '2025-03-05T11:30:29.166Z'
[email protected]: '2026-10-04T03:53:48.117Z'
[email protected]: '2025-03-30T15:40:10.885Z'
[email protected]: '2025-02-14T01:43:48.709Z'
@@ -17173,7 +17036,6 @@ time:
[email protected]: '2026-09-09T17:17:30.281Z'
[email protected]: '2022-09-03T01:07:26.139Z'
[email protected]: '2026-09-27T13:46:24.509Z'
- [email protected]: '2026-10-01T06:46:31.760Z'
[email protected]: '2026-09-25T08:35:02.910Z'
[email protected]: '2026-02-11T22:42:31.720Z'
[email protected]: '2026-03-09T17:23:55.795Z'
diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml
index 21c760525c1..744e1e32130 100644
--- a/pnpm-workspace.yaml
+++ b/pnpm-workspace.yaml
@@ -100,6 +100,7 @@ catalog:
'@orpc/contract': 1.15.7
'@orpc/openapi-client': 1.15.7
'@orpc/tanstack-query': 1.15.7
+ '@pierre/highlights': 0.2.0
'@playwright/test': 1.64.1
'@remixicon/react': 4.9.0
'@sentry/react': 10.76.2
@@ -194,7 +195,6 @@ catalog:
foxact: 0.3.12
fuse.js: 7.5.0
happy-dom: 20.14.6
- hast-util-to-jsx-runtime: 2.3.6
hono: 4.13.13
html-entities: 2.6.0
html-to-image: 1.11.13
@@ -247,7 +247,6 @@ catalog:
scheduler: 0.28.0
server-only: 0.0.1
sharp: 0.35.5
- shiki: 4.5.0
socket.io-client: 4.8.4
sortablejs: 1.15.7
std-semver: 1.0.8
diff --git a/web/app/components/base/markdown-blocks/__tests__/code-block-loading.spec.tsx b/web/app/components/base/markdown-blocks/__tests__/code-block-loading.spec.tsx
index ff1645142c2..63d9606ae34 100644
--- a/web/app/components/base/markdown-blocks/__tests__/code-block-loading.spec.tsx
+++ b/web/app/components/base/markdown-blocks/__tests__/code-block-loading.spec.tsx
@@ -23,9 +23,6 @@ const runtime = vi.hoisted(() => {
}
})
-vi.mock('../shiki-highlight', () => ({
- highlightCode: async ({ code }: { code: string }) => <pre>{code}</pre>,
-}))
vi.mock('echarts-for-react', () => {
runtime.chartLoaded = true
return {
@@ -65,7 +62,11 @@ it('loads chart, music, and SVG runtimes only when their code-block language is
const { rerender, container } = render(
<CodeBlock className="language-typescript">const answer = 42</CodeBlock>,
)
- await waitFor(() => expect(screen.getByText('const answer = 42')).toBeInTheDocument())
+ await waitFor(() =>
+ expect(container.querySelector('.code-line-numbers code')).toHaveTextContent(
+ 'const answer = 42',
+ ),
+ )
expect(runtime.svgLoaded).toBe(false)
expect(runtime.chartLoaded).toBe(false)
expect(runtime.musicLoaded).toBe(false)
diff --git a/web/app/components/base/markdown-blocks/__tests__/code-block.spec.tsx b/web/app/components/base/markdown-blocks/__tests__/code-block.spec.tsx
index aba81e2f23a..b27803e7dc7 100644
--- a/web/app/components/base/markdown-blocks/__tests__/code-block.spec.tsx
+++ b/web/app/components/base/markdown-blocks/__tests__/code-block.spec.tsx
@@ -2,10 +2,7 @@ import { act, render, screen, waitFor } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import { Theme } from '@/types/app'
import { CodeBlock } from '../code-block'
-
-const { mockHighlightCode } = vi.hoisted(() => ({
- mockHighlightCode: vi.fn(),
-}))
+import { highlightCode } from '../code-highlight'
type UseThemeReturn = {
theme: Theme
@@ -73,9 +70,9 @@ vi.mock('@/hooks/use-theme', () => ({
default: () => mockUseTheme(),
}))
-vi.mock('../shiki-highlight', () => ({
- highlightCode: mockHighlightCode,
-}))
+// Keep the real highlighter so tests cover its output; the spy only forces
+// the failure fallback.
+vi.mock('../code-highlight', { spy: true })
vi.mock('../../svg-gallery', () => ({
default: ({ content }: { content: string }) => <div data-testid="svg-renderer">{content}</div>,
@@ -142,15 +139,17 @@ const findEchartsHost = async () => {
return document.querySelector('.echarts-for-react') as HTMLDivElement
}
+const findHighlightedKeyword = async () => {
+ await waitFor(() => {
+ expect(document.querySelector('.code-line-numbers .line span'))!.toBeInTheDocument()
+ })
+ return document.querySelector('.code-line-numbers .line span') as HTMLSpanElement
+}
+
describe('CodeBlock', () => {
beforeEach(() => {
vi.clearAllMocks()
mockUseTheme.mockReturnValue({ theme: Theme.light })
- mockHighlightCode.mockImplementation(async ({ code, language }) => (
- <pre className="shiki">
- <code className={`language-${language}`}>{code}</code>
- </pre>
- ))
consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
consoleWarnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
clientWidthSpy = vi.spyOn(HTMLElement.prototype, 'clientWidth', 'get').mockReturnValue(900)
@@ -218,11 +217,9 @@ describe('CodeBlock', () => {
render(<CodeBlock className="language-javascript">const x = 1;</CodeBlock>)
expect(screen.getByText('JavaScript'))!.toBeInTheDocument()
- await waitFor(() => {
- expect(document.querySelector('code.language-javascript')?.textContent).toContain(
- 'const x = 1;',
- )
- })
+ // github-light keyword color
+ expect(await findHighlightedKeyword()).toHaveStyle({ color: '#d73a49' })
+ expect(document.querySelector('.code-line-numbers code')?.textContent).toBe('const x = 1;')
})
it('should format unknown language labels with capitalized fallback when language is not in map', () => {
@@ -257,22 +254,20 @@ describe('CodeBlock', () => {
render(<CodeBlock className="language-javascript">const y = 2;</CodeBlock>)
expect(screen.getByText('JavaScript'))!.toBeInTheDocument()
- await waitFor(() => {
- expect(document.querySelector('code.language-javascript')?.textContent).toContain(
- 'const y = 2;',
- )
- })
+ // github-dark keyword color
+ expect(await findHighlightedKeyword()).toHaveStyle({ color: '#f97583' })
})
- it('should fall back to plain code block when shiki highlighting fails', async () => {
- mockHighlightCode.mockRejectedValueOnce(new Error('highlight failed'))
+ it('should fall back to plain code block when highlighting fails', async () => {
+ vi.mocked(highlightCode).mockRejectedValueOnce(new Error('highlight failed'))
render(<CodeBlock className="language-javascript">const z = 3;</CodeBlock>)
await waitFor(() => {
- expect(screen.getByText('const z = 3;'))!.toBeInTheDocument()
+ expect(consoleErrorSpy).toHaveBeenCalledWith('Code highlighting failed:', expect.any(Error))
})
- expect(document.querySelector('code.language-javascript')).toBeNull()
+ expect(screen.getByText('const z = 3;'))!.toBeInTheDocument()
+ expect(document.querySelector('.code-line-numbers')).toBeNull()
})
})
diff --git a/web/app/components/base/markdown-blocks/__tests__/shiki-highlight.spec.tsx b/web/app/components/base/markdown-blocks/__tests__/code-highlight.spec.tsx
similarity index 52%
rename from web/app/components/base/markdown-blocks/__tests__/shiki-highlight.spec.tsx
rename to web/app/components/base/markdown-blocks/__tests__/code-highlight.spec.tsx
index 8e6a19e1ffa..a3d34738c53 100644
--- a/web/app/components/base/markdown-blocks/__tests__/shiki-highlight.spec.tsx
+++ b/web/app/components/base/markdown-blocks/__tests__/code-highlight.spec.tsx
@@ -1,42 +1,16 @@
-import type { createSingletonShorthands } from 'shiki/core'
-import type * as shikiCore from 'shiki/core'
+import { codeToTokens } from '@pierre/highlights'
import { renderToStaticMarkup } from 'react-dom/server'
-import { highlightCode } from '../shiki-highlight'
-
-// `shiki/core` only exposes `getSingletonHighlighter` as a property of the
-// shorthand bundle returned by `createSingletonShorthands`, not as a top-level
-// symbol. Derive the accessor signature from the return type so wrapped calls
-// remain correctly typed without importing a non-exported name.
-type SingletonAccessor = ReturnType<typeof createSingletonShorthands>['getSingletonHighlighter']
-
-// Mock the singleton accessor so tests can assert that the WASM engine is not
-// loaded for plain-text fences while still letting supported-language tests
-// observe a real `getSingletonHighlighter` invocation.
-const { mockGetSingletonHighlighter } = vi.hoisted(() => ({
- mockGetSingletonHighlighter: vi.fn(),
-}))
-
-vi.mock('shiki/core', async (importOriginal) => {
- const actual = await importOriginal<typeof shikiCore>()
- const realCreateSingletonShorthands = actual.createSingletonShorthands
- return {
- ...actual,
- createSingletonShorthands: (factory: Parameters<typeof realCreateSingletonShorthands>[0]) => {
- const real = realCreateSingletonShorthands(factory)
- // Wrap real `getSingletonHighlighter` so tests can observe invocations.
- const realGet = real.getSingletonHighlighter
- const wrapped: SingletonAccessor = (...args: Parameters<typeof realGet>) => {
- mockGetSingletonHighlighter(...args)
- return realGet(...args)
- }
- return { getSingletonHighlighter: wrapped }
- },
- }
-})
+import { highlightCode } from '../code-highlight'
+
+// Spy on the real tokenizer so tests can assert that plain-text and unsupported
+// fences never reach the WebAssembly lexers.
+vi.mock('@pierre/highlights', { spy: true })
+
+const mockCodeToTokens = vi.mocked(codeToTokens)
describe('README code highlighting', () => {
beforeEach(() => {
- mockGetSingletonHighlighter.mockReset()
+ mockCodeToTokens.mockClear()
})
it.each(['github-light', 'github-dark'] as const)('highlights dotenv with %s', async (theme) => {
@@ -47,7 +21,8 @@ describe('README code highlighting', () => {
expect(result).toContain('your-api-key')
expect(result).toContain('OPENAI_ORGANIZATION=org-id')
expect(result).toContain('<span style="color:')
- expect(mockGetSingletonHighlighter).toHaveBeenCalledTimes(1)
+ expect(mockCodeToTokens).toHaveBeenCalledTimes(1)
+ expect(mockCodeToTokens.mock.calls[0]![1]).toMatchObject({ lang: 'bash' })
})
it('renders unsupported languages as readable, escaped plain text', async () => {
@@ -61,11 +36,11 @@ describe('README code highlighting', () => {
)
expect(result).toContain('&lt;custom&gt;example&lt;/custom&gt;')
- expect(mockGetSingletonHighlighter).not.toHaveBeenCalled()
+ expect(mockCodeToTokens).not.toHaveBeenCalled()
})
- it.each(['', ' '])(
- 'renders a blank language %j without loading the WASM engine',
+ it.each(['', ' ', 'text', 'TXT'])(
+ 'renders the plain language %j without tokenizing',
async (language) => {
const code = 'plain line one\nplain line two'
const result = renderToStaticMarkup(
@@ -74,7 +49,7 @@ describe('README code highlighting', () => {
expect(result).toContain('plain line one')
expect(result).toContain('plain line two')
- expect(mockGetSingletonHighlighter).not.toHaveBeenCalled()
+ expect(mockCodeToTokens).not.toHaveBeenCalled()
},
)
@@ -92,8 +67,26 @@ describe('README code highlighting', () => {
const document = new DOMParser().parseFromString(result, 'text/html')
expect(document.querySelector('code')?.textContent).toBe(lines.join('\n'))
+ expect(document.querySelectorAll('.line')).toHaveLength(lines.length)
expect(document.querySelector('custom')).toBeNull()
- expect(mockGetSingletonHighlighter).not.toHaveBeenCalled()
+ },
+ )
+
+ it.each(['\n', '\r\n'])(
+ 'renders one line element per highlighted line with %j line endings',
+ async (lineEnding) => {
+ const lines = ['const a = 1', '', ' const b = a']
+ const result = renderToStaticMarkup(
+ await highlightCode({
+ code: lines.join(lineEnding),
+ language: 'ts',
+ theme: 'github-dark',
+ }),
+ )
+ const document = new DOMParser().parseFromString(result, 'text/html')
+
+ expect(document.querySelector('code')?.textContent).toBe(lines.join('\n'))
+ expect(document.querySelectorAll('.line')).toHaveLength(lines.length)
},
)
@@ -104,20 +97,33 @@ describe('README code highlighting', () => {
)
expect(result).toContain('a &lt; b &amp;&amp; b &gt; c &amp; &quot;x&quot;')
- expect(mockGetSingletonHighlighter).not.toHaveBeenCalled()
})
- it('preserves highlighting for bundled language aliases', async () => {
+ it('escapes HTML-sensitive characters in highlighted fences', async () => {
+ const result = renderToStaticMarkup(
+ await highlightCode({
+ code: '<div class="x">&amp;</div>',
+ language: 'html',
+ theme: 'github-light',
+ }),
+ )
+
+ expect(result).not.toContain('<div class="x">')
+ expect(result).toContain('&lt;')
+ expect(result).toContain('&amp;amp;')
+ })
+
+ it.each(['js', 'Python', 'go'])('highlights the bundled language %s', async (language) => {
const result = renderToStaticMarkup(
await highlightCode({
code: 'const count = 1',
- language: 'js',
+ language,
theme: 'github-light',
}),
)
- expect(result).toContain('const')
+ expect(result).toContain('count')
expect(result).toContain('<span style="color:')
- expect(mockGetSingletonHighlighter).toHaveBeenCalledTimes(1)
+ expect(mockCodeToTokens).toHaveBeenCalledTimes(1)
})
})
diff --git a/web/app/components/base/markdown-blocks/code-block.tsx b/web/app/components/base/markdown-blocks/code-block.tsx
index 22faea4b0b4..2bfa44c6a8e 100644
--- a/web/app/components/base/markdown-blocks/code-block.tsx
+++ b/web/app/components/base/markdown-blocks/code-block.tsx
@@ -2,7 +2,7 @@ import type { EChartsOption, EChartsType } from 'echarts'
import type EChartsReact from 'echarts-for-react'
import type { EChartsReactProps } from 'echarts-for-react'
import type { JSX } from 'react'
-import type { CodeTheme } from './shiki-highlight'
+import type { CodeTheme } from './code-highlight'
import { IconButton } from '@langgenius/dify-ui/icon-button'
import { Toggle } from '@langgenius/dify-ui/toggle'
import dynamic from 'next/dynamic'
@@ -21,7 +21,7 @@ import CopyIcon from '@/app/components/base/copy-icon'
import ErrorBoundary from '@/app/components/base/markdown/error-boundary'
import useTheme from '@/hooks/use-theme'
import { Theme } from '@/types/app'
-import { highlightCode } from './shiki-highlight'
+import { highlightCode } from './code-highlight'
const MermaidDiagram = dynamic(
() => import('./mermaid-diagram').then((module) => module.MermaidDiagram),
@@ -76,7 +76,7 @@ const getCorrectCapitalizationLanguageName = (language: string) => {
// visit https://reactjs.org/docs/error-decoder.html?invariant=185 for the full message
// or use the non-minified dev environment for full errors and additional helpful warnings.
-const ShikiCodeBlock = memo(
+const HighlightedCodeBlock = memo(
({
code,
language,
@@ -102,7 +102,7 @@ const ShikiCodeBlock = memo(
if (!cancelled) setNodes(result)
})
.catch((error) => {
- console.error('Shiki highlighting failed:', error)
+ console.error('Code highlighting failed:', error)
if (!cancelled) setNodes(undefined)
})
@@ -135,14 +135,14 @@ const ShikiCodeBlock = memo(
borderBottomRightRadius: '10px',
overflow: 'auto',
}}
- className="shiki-line-numbers [&_pre]:m-0! [&_pre]:rounded-t-none! [&_pre]:rounded-b-[10px]! [&_pre]:bg-components-input-bg-normal! [&_pre]:py-2!"
+ className="code-line-numbers [&_pre]:m-0! [&_pre]:rounded-t-none! [&_pre]:rounded-b-[10px]! [&_pre]:bg-components-input-bg-normal! [&_pre]:py-2!"
>
{nodes}
</div>
)
},
)
-ShikiCodeBlock.displayName = 'ShikiCodeBlock'
+HighlightedCodeBlock.displayName = 'HighlightedCodeBlock'
// Define ECharts event parameter types
type EChartsEventParams = {
@@ -525,7 +525,7 @@ export const CodeBlock: any = memo(({ inline, className, children = '', ...props
)
default:
return (
- <ShikiCodeBlock
+ <HighlightedCodeBlock
code={content}
language={match?.[1] || 'text'}
theme={isDarkMode ? 'github-dark' : 'github-light'}
diff --git a/web/app/components/base/markdown-blocks/code-highlight.tsx b/web/app/components/base/markdown-blocks/code-highlight.tsx
new file mode 100644
index 00000000000..d28aae58bc3
--- /dev/null
+++ b/web/app/components/base/markdown-blocks/code-highlight.tsx
@@ -0,0 +1,86 @@
+import type { ThemedToken } from '@pierre/highlights'
+import type { CSSProperties, JSX } from 'react'
+import { Fragment } from 'react'
+
+const themes = {
+ 'github-dark': () => import('@pierre/highlights/themes/github-dark'),
+ 'github-light': () => import('@pierre/highlights/themes/github-light'),
+}
+
+export type CodeTheme = keyof typeof themes
+
+// Highlights bundles a fixed lexer set. Map README fence names it lacks to a
+// close lexer; other unsupported names render as plain text.
+const languageAliases: Record<string, string> = {
+ dotenv: 'bash',
+}
+
+const plainLanguages = new Set(['', 'plain', 'plaintext', 'text', 'txt'])
+
+type HighlightCodeOptions = {
+ code: string
+ language: string
+ theme: CodeTheme
+}
+
+type CodeLine = Pick<ThemedToken, 'content' | 'color' | 'fontStyle'>[]
+
+const getTokenStyle = ({ color, fontStyle = 0 }: CodeLine[number]): CSSProperties | undefined => {
+ if (!color && !fontStyle) return undefined
+ return {
+ color,
+ // Highlights only emits Shiki's italic (1) and bold (2) flags.
+ fontStyle: fontStyle & 1 ? 'italic' : undefined,
+ fontWeight: fontStyle & 2 ? 'bold' : undefined,
+ }
+}
+
+const renderLines = (lines: CodeLine[], style?: CSSProperties) => (
+ <pre style={style}>
+ <code>
+ {lines.map((tokens, lineIndex) => (
+ // oxlint-disable-next-line react/no-array-index-key -- Lines are positional and re-render together.
+ <Fragment key={lineIndex}>
+ {lineIndex > 0 && '\n'}
+ {/* The code-block gutter uses .line::before for its line numbers. */}
+ <span className="line">
+ {tokens.map((token, tokenIndex) => (
+ // oxlint-disable-next-line react/no-array-index-key -- Tokens are positional within their line.
+ <span key={tokenIndex} style={getTokenStyle(token)}>
+ {token.content}
+ </span>
+ ))}
+ </span>
+ </Fragment>
+ ))}
+ </code>
+ </pre>
+)
+
+const renderPlainText = (code: string) =>
+ renderLines(code.split(/\r?\n/).map((content) => [{ content }]))
+
+export const highlightCode = async ({
+ code,
+ language,
+ theme,
+}: HighlightCodeOptions): Promise<JSX.Element> => {
+ const normalizedLanguage = language.trim().toLowerCase()
+ const lang = languageAliases[normalizedLanguage] ?? normalizedLanguage
+
+ // Plain text does not need syntax highlighting, so skip loading the
+ // WebAssembly lexers for it (see #42943).
+ if (plainLanguages.has(lang)) return renderPlainText(code)
+
+ const [{ codeToTokens, isSupportedLanguage }, { default: themeDefinition }] = await Promise.all([
+ import('@pierre/highlights'),
+ themes[theme](),
+ ])
+
+ // README fences may name languages outside the bundled lexers. Keep them
+ // readable instead of letting Highlights throw a RangeError.
+ if (!isSupportedLanguage(lang)) return renderPlainText(code)
+
+ const { tokens, fg, bg } = codeToTokens(code, { lang, theme: themeDefinition })
+ return renderLines(tokens, { backgroundColor: bg, color: fg })
+}
diff --git a/web/app/components/base/markdown-blocks/shiki-highlight.tsx b/web/app/components/base/markdown-blocks/shiki-highlight.tsx
deleted file mode 100644
index b3085dbd81b..00000000000
--- a/web/app/components/base/markdown-blocks/shiki-highlight.tsx
+++ /dev/null
@@ -1,97 +0,0 @@
-import type { JSX } from 'react'
-import { toJsxRuntime } from 'hast-util-to-jsx-runtime'
-import { Fragment } from 'react'
-import { jsx, jsxs } from 'react/jsx-runtime'
-import { createBundledHighlighter, createSingletonShorthands } from 'shiki/core'
-import { createOnigurumaEngine } from 'shiki/engine/oniguruma'
-import { codeLanguages } from './shiki-languages'
-
-const createHighlighter = createBundledHighlighter({
- langs: codeLanguages,
- themes: {
- 'github-dark': () => import('shiki/themes/github-dark.mjs'),
- 'github-light': () => import('shiki/themes/github-light.mjs'),
- },
- engine: () => createOnigurumaEngine(import('shiki/wasm')),
-})
-
-const { getSingletonHighlighter } = createSingletonShorthands(createHighlighter)
-
-export type CodeTheme = 'github-dark' | 'github-light'
-
-type HighlightCodeOptions = {
- code: string
- language: string
- theme: CodeTheme
-}
-
-export const highlightCode = async ({
- code,
- language,
- theme,
-}: HighlightCodeOptions): Promise<JSX.Element> => {
- const normalizedLanguage = language.trim().toLowerCase()
- const lang =
- normalizedLanguage === 'dotenv' || Object.hasOwn(codeLanguages, normalizedLanguage)
- ? normalizedLanguage
- : 'text'
-
- // Plain text and unsupported languages do not need syntax highlighting, so
- // skip `getSingletonHighlighter` to avoid loading the WASM engine for them
- // (see #42943). The shiki engine is still loaded on the first real highlight
- // call, so subsequent supported-language code blocks render normally.
- if (lang === 'text') {
- // Hand-built minimal HAST tree; typed via `toJsxRuntime`'s own parameter
- // so we don't need a top-level `hast` import (the shipped `@types/hast`
- // transitively pulls in `hast` module declarations that we cannot resolve).
- const plainHast: Parameters<typeof toJsxRuntime>[0] = {
- type: 'root',
- children: [
- {
- type: 'element',
- tagName: 'pre',
- properties: {},
- children: [
- {
- type: 'element',
- tagName: 'code',
- properties: {},
- // The code-block gutter uses .line::before for its line numbers.
- children: code.split(/\r?\n/).flatMap((line, index) => [
- { type: 'text' as const, value: index === 0 ? '' : '\n' },
- {
- type: 'element' as const,
- tagName: 'span',
- properties: { className: ['line'] },
- children: [{ type: 'text' as const, value: line }],
- },
- ]),
- },
- ],
- },
- ],
- }
- return toJsxRuntime(plainHast, {
- Fragment,
- jsx,
- jsxs,
- }) as JSX.Element
- }
-
- // README fences may name languages outside the web bundle. Load dotenv on
- // demand and keep unknown languages readable without throwing an error.
- const highlighter = await getSingletonHighlighter({
- langs: lang === 'dotenv' ? [(await import('shiki/langs/dotenv.mjs')).default] : [lang],
- themes: [theme],
- })
- const hast = highlighter.codeToHast(code, {
- lang,
- theme,
- })
-
- return toJsxRuntime(hast, {
- Fragment,
- jsx,
- jsxs,
- }) as JSX.Element
-}
diff --git a/web/app/components/base/markdown-blocks/shiki-languages.ts b/web/app/components/base/markdown-blocks/shiki-languages.ts
deleted file mode 100644
index 5e2b8cb64df..00000000000
--- a/web/app/components/base/markdown-blocks/shiki-languages.ts
+++ /dev/null
@@ -1,82 +0,0 @@
-// Keep the existing Shiki web language set and aliases without importing its all-theme bundle.
-export const codeLanguages = {
- 'angular-html': () => import('shiki/langs/angular-html.mjs'),
- 'angular-ts': () => import('shiki/langs/angular-ts.mjs'),
- astro: () => import('shiki/langs/astro.mjs'),
- blade: () => import('shiki/langs/blade.mjs'),
- c: () => import('shiki/langs/c.mjs'),
- coffee: () => import('shiki/langs/coffee.mjs'),
- coffeescript: () => import('shiki/langs/coffee.mjs'),
- cpp: () => import('shiki/langs/cpp.mjs'),
- 'c++': () => import('shiki/langs/cpp.mjs'),
- css: () => import('shiki/langs/css.mjs'),
- csv: () => import('shiki/langs/csv.mjs'),
- glsl: () => import('shiki/langs/glsl.mjs'),
- graphql: () => import('shiki/langs/graphql.mjs'),
- gql: () => import('shiki/langs/graphql.mjs'),
- haml: () => import('shiki/langs/haml.mjs'),
- handlebars: () => import('shiki/langs/handlebars.mjs'),
- hbs: () => import('shiki/langs/handlebars.mjs'),
- html: () => import('shiki/langs/html.mjs'),
- 'html-derivative': () => import('shiki/langs/html-derivative.mjs'),
- http: () => import('shiki/langs/http.mjs'),
- hurl: () => import('shiki/langs/hurl.mjs'),
- imba: () => import('shiki/langs/imba.mjs'),
- java: () => import('shiki/langs/java.mjs'),
- javascript: () => import('shiki/langs/javascript.mjs'),
- js: () => import('shiki/langs/javascript.mjs'),
- cjs: () => import('shiki/langs/javascript.mjs'),
- mjs: () => import('shiki/langs/javascript.mjs'),
- jinja: () => import('shiki/langs/jinja.mjs'),
- jison: () => import('shiki/langs/jison.mjs'),
- json: () => import('shiki/langs/json.mjs'),
- json5: () => import('shiki/langs/json5.mjs'),
- jsonc: () => import('shiki/langs/jsonc.mjs'),
- jsonl: () => import('shiki/langs/jsonl.mjs'),
- jsx: () => import('shiki/langs/jsx.mjs'),
- julia: () => import('shiki/langs/julia.mjs'),
- jl: () => import('shiki/langs/julia.mjs'),
- less: () => import('shiki/langs/less.mjs'),
- markdown: () => import('shiki/langs/markdown.mjs'),
- md: () => import('shiki/langs/markdown.mjs'),
- marko: () => import('shiki/langs/marko.mjs'),
- mdc: () => import('shiki/langs/mdc.mjs'),
- mdx: () => import('shiki/langs/mdx.mjs'),
- php: () => import('shiki/langs/php.mjs'),
- postcss: () => import('shiki/langs/postcss.mjs'),
- pug: () => import('shiki/langs/pug.mjs'),
- jade: () => import('shiki/langs/pug.mjs'),
- python: () => import('shiki/langs/python.mjs'),
- py: () => import('shiki/langs/python.mjs'),
- r: () => import('shiki/langs/r.mjs'),
- regexp: () => import('shiki/langs/regexp.mjs'),
- regex: () => import('shiki/langs/regexp.mjs'),
- sass: () => import('shiki/langs/sass.mjs'),
- scss: () => import('shiki/langs/scss.mjs'),
- shellscript: () => import('shiki/langs/shellscript.mjs'),
- bash: () => import('shiki/langs/shellscript.mjs'),
- sh: () => import('shiki/langs/shellscript.mjs'),
- shell: () => import('shiki/langs/shellscript.mjs'),
- zsh: () => import('shiki/langs/shellscript.mjs'),
- smithy: () => import('shiki/langs/smithy.mjs'),
- sql: () => import('shiki/langs/sql.mjs'),
- stylus: () => import('shiki/langs/stylus.mjs'),
- styl: () => import('shiki/langs/stylus.mjs'),
- svelte: () => import('shiki/langs/svelte.mjs'),
- 'ts-tags': () => import('shiki/langs/ts-tags.mjs'),
- lit: () => import('shiki/langs/ts-tags.mjs'),
- tsx: () => import('shiki/langs/tsx.mjs'),
- typescript: () => import('shiki/langs/typescript.mjs'),
- ts: () => import('shiki/langs/typescript.mjs'),
- cts: () => import('shiki/langs/typescript.mjs'),
- mts: () => import('shiki/langs/typescript.mjs'),
- vue: () => import('shiki/langs/vue.mjs'),
- 'vue-html': () => import('shiki/langs/vue-html.mjs'),
- 'vue-vine': () => import('shiki/langs/vue-vine.mjs'),
- wasm: () => import('shiki/langs/wasm.mjs'),
- wgsl: () => import('shiki/langs/wgsl.mjs'),
- wit: () => import('shiki/langs/wit.mjs'),
- xml: () => import('shiki/langs/xml.mjs'),
- yaml: () => import('shiki/langs/yaml.mjs'),
- yml: () => import('shiki/langs/yaml.mjs'),
-}
diff --git a/web/app/components/datasets/documents/detail/completed/segment-card/__tests__/index.spec.tsx b/web/app/components/datasets/documents/detail/completed/segment-card/__tests__/index.spec.tsx
index 38325797cfb..d5854243ff2 100644
--- a/web/app/components/datasets/documents/detail/completed/segment-card/__tests__/index.spec.tsx
+++ b/web/app/components/datasets/documents/detail/completed/segment-card/__tests__/index.spec.tsx
@@ -87,7 +87,7 @@ vi.mock('@/app/components/datasets/common/image-list', () => ({
),
}))
-// Markdown uses next/dynamic and shiki (ESM)
+// Markdown loads its code blocks through next/dynamic
vi.mock('@/app/components/base/markdown', () => ({
Markdown: ({ content, className }: { content: string; className?: string }) => (
<div data-testid="markdown" className={`markdown-body ${className || ''}`}>
diff --git a/web/app/styles/tailwind-core.css b/web/app/styles/tailwind-core.css
index fe791364be9..50a162384f2 100644
--- a/web/app/styles/tailwind-core.css
+++ b/web/app/styles/tailwind-core.css
@@ -150,11 +150,11 @@
background-clip: text;
}
- /* Shiki code block line numbers */
- .shiki-line-numbers code {
+ /* Highlighted code block line numbers */
+ .code-line-numbers code {
counter-reset: line;
}
- .shiki-line-numbers .line::before {
+ .code-line-numbers .line::before {
counter-increment: line;
content: counter(line);
display: inline-block;
diff --git a/web/package.json b/web/package.json
index fc7d675aa3d..4992a9f2846 100644
--- a/web/package.json
+++ b/web/package.json
@@ -59,6 +59,7 @@
"@orpc/contract": "catalog:",
"@orpc/openapi-client": "catalog:",
"@orpc/tanstack-query": "catalog:",
+ "@pierre/highlights": "catalog:",
"@remixicon/react": "catalog:",
"@sentry/react": "catalog:",
"@streamdown/math": "catalog:",
@@ -89,7 +90,6 @@
"fast-deep-equal": "catalog:",
"foxact": "catalog:",
"fuse.js": "catalog:",
- "hast-util-to-jsx-runtime": "catalog:",
"html-entities": "catalog:",
"html-to-image": "catalog:",
"i18next": "catalog:",
@@ -128,7 +128,6 @@
"remark-directive": "catalog:",
"scheduler": "catalog:",
"server-only": "catalog:",
- "shiki": "catalog:",
"socket.io-client": "catalog:",
"sortablejs": "catalog:",
"std-semver": "catalog:",