Skip to content

Your lightweight open-source personal agent.

fix(webui): parse CJK emphasis at delimiter boundaries

chengyongru51a0fc83a2d2
diff --git a/webui/browser/fixture.tsx b/webui/browser/fixture.tsx
index 20e9d2d35..4f0f51de7 100644
--- a/webui/browser/fixture.tsx
+++ b/webui/browser/fixture.tsx
@@ -23,6 +23,7 @@ import { NewChatSkeletonFixture } from "./new-chat-skeleton-fixture";
import "../src/globals.css";
import { DiagramFixture } from "./diagram-fixture";
import { TypographyFixture } from "./typography-fixture";
+import { MarkdownEmphasisFixture } from "./markdown-emphasis-fixture";
import { HistoryBoundaryFixture } from "./history-boundary-fixture";
import { ChatFailureFixture } from "./chat-failure-fixture";
import { ThreadLoadingFixture } from "./thread-loading-fixture";
@@ -175,6 +176,8 @@ createRoot(document.getElementById("root")!).render(<TooltipProvider>
? <SidebarActivityFixture />
: parameters.get("case") === "chat-failure"
? <ChatFailureFixture parameters={parameters} />
+ : parameters.get("case") === "markdown-emphasis"
+ ? <MarkdownEmphasisFixture />
: parameters.get("case") === "typography"
? <ClientProvider client={railClient} token="fixture"><TypographyFixture /></ClientProvider>
: parameters.get("case") === "history-hydration"
diff --git a/webui/browser/markdown-emphasis-fixture.tsx b/webui/browser/markdown-emphasis-fixture.tsx
new file mode 100644
index 000000000..9b9a75b46
--- /dev/null
+++ b/webui/browser/markdown-emphasis-fixture.tsx
@@ -0,0 +1,21 @@
+import { useState } from "react";
+
+import { MarkdownText } from "@/components/MarkdownText";
+
+const source = [
+ "**`ThreadTranscript` 已经朝这个方向走了一步:它把正文、来源覆盖和分页接纳集中起来,因此可以看作一个隐式的状态转换模型。**但“集中到一个类”不等于已经有清晰的状态机。",
+ "前文**「重点」**后文。**参考 [说明](https://example.com) 的结论。**继续阅读。",
+ "**日本語。**続き。**한국어(설명)**입니다。",
+ "**中文。** 继续。**English.** Next.",
+ "**边界说明: **issue",
+].join("\n\n");
+
+export function MarkdownEmphasisFixture() {
+ const [end, setEnd] = useState(0);
+ const [streaming, setStreaming] = useState(true);
+ return <main className="mx-auto max-w-3xl p-6">
+ <button onClick={() => setEnd((value) => Math.min(source.length, value + 1))}>Next character</button>
+ <button onClick={() => { setEnd(source.length); setStreaming(false); }}>Complete</button>
+ <div data-testid="reply"><MarkdownText streaming={streaming} preserveStreamingLayout>{source.slice(0, end)}</MarkdownText></div>
+ </main>;
+}
diff --git a/webui/browser/markdown-emphasis.spec.ts b/webui/browser/markdown-emphasis.spec.ts
new file mode 100644
index 000000000..dab0ccfb5
--- /dev/null
+++ b/webui/browser/markdown-emphasis.spec.ts
@@ -0,0 +1,23 @@
+import { expect, test } from "@playwright/test";
+
+for (const width of [390, 1440]) {
+ test(`CJK emphasis retains inline semantics through completion at ${width}px`, async ({ page }) => {
+ await page.setViewportSize({ width, height: 900 });
+ await page.goto("/browser/fixture.html?case=markdown-emphasis");
+ // Reveal the inline code while its enclosing emphasis is still incomplete.
+ for (let index = 0; index < 24; index++) {
+ await page.getByRole("button", { name: "Next character", exact: true }).click();
+ }
+ const reply = page.getByTestId("reply");
+ await expect(reply.locator("code")).toHaveText("ThreadTranscript");
+ await page.getByRole("button", { name: "Complete", exact: true }).click();
+ await expect(reply.locator("strong code")).toHaveText("ThreadTranscript");
+ await expect(reply.locator("strong a")).toHaveText("说明");
+ await expect(reply.locator("strong a")).toHaveAttribute("href", "https://example.com");
+ await expect(reply.locator("strong")).toHaveCount(7);
+ await expect(reply.locator("p").last()).toHaveText("**边界说明: **issue");
+ await expect(reply.locator("p").first()).toContainText("但“集中到一个类”不等于已经有清晰的状态机。");
+ await expect(reply.locator("strong").first()).not.toContainText("但“集中到一个类”");
+ await expect(reply).toBeVisible();
+ });
+}
diff --git a/webui/bun.lock b/webui/bun.lock
index aeb72bb69..64cba7e29 100644
--- a/webui/bun.lock
+++ b/webui/bun.lock
@@ -30,6 +30,7 @@
"react-syntax-highlighter": "^15.6.1",
"rehype-katex": "^7.0.1",
"remark-breaks": "^4.0.0",
+ "remark-cjk-friendly": "2.3.3",
"remark-gfm": "^4.0.0",
"remark-math": "^6.0.0",
"remark-parse": "^11.0.0",
@@ -639,6 +640,8 @@
"get-caller-file": ["[email protected]", "", {}, "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg=="],
+ "get-east-asian-width": ["[email protected]", "https://mirrors.tencent.com/npm/get-east-asian-width/-/get-east-asian-width-1.7.0.tgz", {}, "sha512-XjH1AECxf0giL2V1aU8vKyRR2ppRUb5c0EvT7zuJTokQ74bNo52zOtghqdWIqrhUD79fo3x0WfKZdOqxF6LG1Q=="],
+
"get-nonce": ["[email protected]", "", {}, "sha512-FJhYRoDaiatfEkUK8HKlicmu/3SGFD51q3itKDGoSTysQJBnfOcxU5GxnhE1E6soB76MbT0MBtnKJuXyAx+96Q=="],
"glob": ["[email protected]", "", { "dependencies": { "foreground-child": "^3.1.0", "jackspeak": "^3.1.2", "minimatch": "^9.0.4", "minipass": "^7.1.2", "package-json-from-dist": "^1.0.0", "path-scurry": "^1.11.1" }, "bin": { "glob": "dist/esm/bin.mjs" } }, "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg=="],
@@ -819,7 +822,9 @@
"mdast-util-to-hast": ["[email protected]", "", { "dependencies": { "@types/hast": "^3.0.0", "@types/mdast": "^4.0.0", "@ungap/structured-clone": "^1.0.0", "devlop": "^1.0.0", "micromark-util-sanitize-uri": "^2.0.0", "trim-lines": "^3.0.0", "unist-util-position": "^5.0.0", "unist-util-visit": "^5.0.0", "vfile": "^6.0.0" } }, "sha512-cctsq2wp5vTsLIcaymblUriiTcZd0CwWtCbLvrOzYCDZoWyMNV8sZ7krj09FSnsiJi3WVsHLM4k6Dq/yaPyCXA=="],
- "mdast-util-to-markdown": ["[email protected]", "", { "dependencies": { "@types/mdast": "^4.0.0", "@types/unist": "^3.0.0", "longest-streak": "^3.0.0", "mdast-util-phrasing": "^4.0.0", "mdast-util-to-string": "^4.0.0", "micromark-util-classify-character": "^2.0.0", "micromark-util-decode-string": "^2.0.0", "unist-util-visit": "^5.0.0", "zwitch": "^2.0.0" } }, "sha512-xj68wMTvGXVOKonmog6LwyJKrYXZPvlwabaryTjLh9LuvovB/KAH+kvi8Gjj+7rJjsFi23nkUxRQv1KqSroMqA=="],
+ "mdast-util-to-markdown": ["[email protected]", "https://mirrors.tencent.com/npm/mdast-util-to-markdown/-/mdast-util-to-markdown-2.2.0.tgz", { "dependencies": { "@types/mdast": "^4.0.0", "@types/unist": "^3.0.0", "longest-streak": "^3.0.0", "mdast-util-phrasing": "^4.0.0", "mdast-util-to-string": "^4.0.0", "micromark-util-character": "^2.0.0", "micromark-util-classify-character": "^2.0.0", "micromark-util-decode-string": "^2.0.0", "micromark-util-html-tag-name": "^2.0.0", "unist-util-visit": "^5.0.0", "zwitch": "^2.0.0" } }, "sha512-2Jn/ADrHFFyyniBMi0bncTbJqU78K3/TEYvzvwrWWzD9MYPUa/Gm4r3nOeKFQg/PDM5BsbDYdc8ackW4NTY6NA=="],
+
+ "mdast-util-to-markdown-cjk-friendly": ["[email protected]", "https://mirrors.tencent.com/npm/mdast-util-to-markdown-cjk-friendly/-/mdast-util-to-markdown-cjk-friendly-1.0.1.tgz", { "dependencies": { "mdast-util-to-markdown": "^2.1.3", "micromark-extension-cjk-friendly-util": "3.0.1", "micromark-util-character": "^2.1.1", "micromark-util-classify-character": "^2.0.1", "micromark-util-html-tag-name": "^2.0.1" }, "peerDependencies": { "@types/mdast": "*" }, "optionalPeers": ["@types/mdast"] }, "sha512-J3njpXAKUXGOa4k5pUwxHV0yuO4k5DgCrHGPady8xCFutjDr31UmuSaLuuxPLJNBO3336dX+CtB4GnNhgl0KKw=="],
"mdast-util-to-string": ["[email protected]", "", { "dependencies": { "@types/mdast": "^4.0.0" } }, "sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg=="],
@@ -831,6 +836,10 @@
"micromark-core-commonmark": ["[email protected]", "", { "dependencies": { "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "micromark-factory-destination": "^2.0.0", "micromark-factory-label": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-factory-title": "^2.0.0", "micromark-factory-whitespace": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-classify-character": "^2.0.0", "micromark-util-html-tag-name": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-resolve-all": "^2.0.0", "micromark-util-subtokenize": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg=="],
+ "micromark-extension-cjk-friendly": ["[email protected]", "https://mirrors.tencent.com/npm/micromark-extension-cjk-friendly/-/micromark-extension-cjk-friendly-2.0.2.tgz", { "dependencies": { "devlop": "^1.1.0", "micromark-extension-cjk-friendly-util": "3.0.1", "micromark-util-chunked": "^2.0.1", "micromark-util-resolve-all": "^2.0.1", "micromark-util-symbol": "^2.0.1" }, "peerDependencies": { "micromark": "^4.0.0", "micromark-util-types": "^2.0.0" }, "optionalPeers": ["micromark-util-types"] }, "sha512-/BqZNCkDWg3Hj9zEEcRitZ5uMF3X78qmBFuP17oAjnVDZ0+H1GHiDKMJZDpJsXasCHEn/kzl6QHozUiWgCHtPQ=="],
+
+ "micromark-extension-cjk-friendly-util": ["[email protected]", "https://mirrors.tencent.com/npm/micromark-extension-cjk-friendly-util/-/micromark-extension-cjk-friendly-util-3.0.1.tgz", { "dependencies": { "get-east-asian-width": "^1.4.0", "micromark-util-character": "^2.1.1", "micromark-util-symbol": "^2.0.1" } }, "sha512-GcbXqTTHOsiZHyF753oIddP/J2eH8j9zpyQPhkof6B2JNxfEJabnQqxbCgzJNuNes0Y2jTNJ3LiYPSXr6eJA8w=="],
+
"micromark-extension-gfm": ["[email protected]", "", { "dependencies": { "micromark-extension-gfm-autolink-literal": "^2.0.0", "micromark-extension-gfm-footnote": "^2.0.0", "micromark-extension-gfm-strikethrough": "^2.0.0", "micromark-extension-gfm-table": "^2.0.0", "micromark-extension-gfm-tagfilter": "^2.0.0", "micromark-extension-gfm-task-list-item": "^2.0.0", "micromark-util-combine-extensions": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-vsKArQsicm7t0z2GugkCKtZehqUm31oeGBV/KVSorWSy8ZlNAv7ytjFhvaryUiCUJYqs+NoE6AFhpQvBTM6Q4w=="],
"micromark-extension-gfm-autolink-literal": ["[email protected]", "", { "dependencies": { "micromark-util-character": "^2.0.0", "micromark-util-sanitize-uri": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-oOg7knzhicgQ3t4QCjCWgTmfNhvQbDDnJeVu9v81r7NltNCVmhPy1fJRX27pISafdjL+SVc4d3l48Gb6pbRypw=="],
@@ -1021,6 +1030,8 @@
"remark-breaks": ["[email protected]", "", { "dependencies": { "@types/mdast": "^4.0.0", "mdast-util-newline-to-break": "^2.0.0", "unified": "^11.0.0" } }, "sha512-IjEjJOkH4FuJvHZVIW0QCDWxcG96kCq7An/KVH2NfJe6rKZU2AsHeB3OEjPNRxi4QC34Xdx7I2KGYn6IpT7gxQ=="],
+ "remark-cjk-friendly": ["[email protected]", "https://mirrors.tencent.com/npm/remark-cjk-friendly/-/remark-cjk-friendly-2.3.3.tgz", { "dependencies": { "mdast-util-to-markdown-cjk-friendly": "1.0.1", "micromark-extension-cjk-friendly": "2.0.2" }, "peerDependencies": { "@types/mdast": "^4.0.0", "unified": "^11.0.0" }, "optionalPeers": ["@types/mdast"] }, "sha512-tx+udvO1M3Pc36Mw4rXDWIFpeIqCzFdbmHq+IjteIEqnKkEjVACYR9FfUqCTDaKEjbjQQ1XlDq7tzRvuXw6Uzg=="],
+
"remark-gfm": ["[email protected]", "", { "dependencies": { "@types/mdast": "^4.0.0", "mdast-util-gfm": "^3.0.0", "micromark-extension-gfm": "^3.0.0", "remark-parse": "^11.0.0", "remark-stringify": "^11.0.0", "unified": "^11.0.0" } }, "sha512-1quofZ2RQ9EWdeN34S79+KExV1764+wCUGop5CPL1WGdD0ocPpu91lzPGbwWMECpEpd42kJGQwzRfyov9j4yNg=="],
"remark-math": ["[email protected]", "", { "dependencies": { "@types/mdast": "^4.0.0", "mdast-util-math": "^3.0.0", "micromark-extension-math": "^3.0.0", "unified": "^11.0.0" } }, "sha512-MMqgnP74Igy+S3WwnhQ7kqGlEerTETXMvJhrUzDikVZ2/uogJCb+WHUg97hK9/jcfc0dkD73s3LN8zU49cTEtA=="],
@@ -1271,8 +1282,26 @@
"mdast-util-find-and-replace/escape-string-regexp": ["[email protected]", "", {}, "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw=="],
+ "mdast-util-gfm/mdast-util-to-markdown": ["[email protected]", "", { "dependencies": { "@types/mdast": "^4.0.0", "@types/unist": "^3.0.0", "longest-streak": "^3.0.0", "mdast-util-phrasing": "^4.0.0", "mdast-util-to-string": "^4.0.0", "micromark-util-classify-character": "^2.0.0", "micromark-util-decode-string": "^2.0.0", "unist-util-visit": "^5.0.0", "zwitch": "^2.0.0" } }, "sha512-xj68wMTvGXVOKonmog6LwyJKrYXZPvlwabaryTjLh9LuvovB/KAH+kvi8Gjj+7rJjsFi23nkUxRQv1KqSroMqA=="],
+
+ "mdast-util-gfm-footnote/mdast-util-to-markdown": ["[email protected]", "", { "dependencies": { "@types/mdast": "^4.0.0", "@types/unist": "^3.0.0", "longest-streak": "^3.0.0", "mdast-util-phrasing": "^4.0.0", "mdast-util-to-string": "^4.0.0", "micromark-util-classify-character": "^2.0.0", "micromark-util-decode-string": "^2.0.0", "unist-util-visit": "^5.0.0", "zwitch": "^2.0.0" } }, "sha512-xj68wMTvGXVOKonmog6LwyJKrYXZPvlwabaryTjLh9LuvovB/KAH+kvi8Gjj+7rJjsFi23nkUxRQv1KqSroMqA=="],
+
+ "mdast-util-gfm-strikethrough/mdast-util-to-markdown": ["[email protected]", "", { "dependencies": { "@types/mdast": "^4.0.0", "@types/unist": "^3.0.0", "longest-streak": "^3.0.0", "mdast-util-phrasing": "^4.0.0", "mdast-util-to-string": "^4.0.0", "micromark-util-classify-character": "^2.0.0", "micromark-util-decode-string": "^2.0.0", "unist-util-visit": "^5.0.0", "zwitch": "^2.0.0" } }, "sha512-xj68wMTvGXVOKonmog6LwyJKrYXZPvlwabaryTjLh9LuvovB/KAH+kvi8Gjj+7rJjsFi23nkUxRQv1KqSroMqA=="],
+
+ "mdast-util-gfm-table/mdast-util-to-markdown": ["[email protected]", "", { "dependencies": { "@types/mdast": "^4.0.0", "@types/unist": "^3.0.0", "longest-streak": "^3.0.0", "mdast-util-phrasing": "^4.0.0", "mdast-util-to-string": "^4.0.0", "micromark-util-classify-character": "^2.0.0", "micromark-util-decode-string": "^2.0.0", "unist-util-visit": "^5.0.0", "zwitch": "^2.0.0" } }, "sha512-xj68wMTvGXVOKonmog6LwyJKrYXZPvlwabaryTjLh9LuvovB/KAH+kvi8Gjj+7rJjsFi23nkUxRQv1KqSroMqA=="],
+
+ "mdast-util-gfm-task-list-item/mdast-util-to-markdown": ["[email protected]", "", { "dependencies": { "@types/mdast": "^4.0.0", "@types/unist": "^3.0.0", "longest-streak": "^3.0.0", "mdast-util-phrasing": "^4.0.0", "mdast-util-to-string": "^4.0.0", "micromark-util-classify-character": "^2.0.0", "micromark-util-decode-string": "^2.0.0", "unist-util-visit": "^5.0.0", "zwitch": "^2.0.0" } }, "sha512-xj68wMTvGXVOKonmog6LwyJKrYXZPvlwabaryTjLh9LuvovB/KAH+kvi8Gjj+7rJjsFi23nkUxRQv1KqSroMqA=="],
+
+ "mdast-util-math/mdast-util-to-markdown": ["[email protected]", "", { "dependencies": { "@types/mdast": "^4.0.0", "@types/unist": "^3.0.0", "longest-streak": "^3.0.0", "mdast-util-phrasing": "^4.0.0", "mdast-util-to-string": "^4.0.0", "micromark-util-classify-character": "^2.0.0", "micromark-util-decode-string": "^2.0.0", "unist-util-visit": "^5.0.0", "zwitch": "^2.0.0" } }, "sha512-xj68wMTvGXVOKonmog6LwyJKrYXZPvlwabaryTjLh9LuvovB/KAH+kvi8Gjj+7rJjsFi23nkUxRQv1KqSroMqA=="],
+
+ "mdast-util-mdx-expression/mdast-util-to-markdown": ["[email protected]", "", { "dependencies": { "@types/mdast": "^4.0.0", "@types/unist": "^3.0.0", "longest-streak": "^3.0.0", "mdast-util-phrasing": "^4.0.0", "mdast-util-to-string": "^4.0.0", "micromark-util-classify-character": "^2.0.0", "micromark-util-decode-string": "^2.0.0", "unist-util-visit": "^5.0.0", "zwitch": "^2.0.0" } }, "sha512-xj68wMTvGXVOKonmog6LwyJKrYXZPvlwabaryTjLh9LuvovB/KAH+kvi8Gjj+7rJjsFi23nkUxRQv1KqSroMqA=="],
+
+ "mdast-util-mdx-jsx/mdast-util-to-markdown": ["[email protected]", "", { "dependencies": { "@types/mdast": "^4.0.0", "@types/unist": "^3.0.0", "longest-streak": "^3.0.0", "mdast-util-phrasing": "^4.0.0", "mdast-util-to-string": "^4.0.0", "micromark-util-classify-character": "^2.0.0", "micromark-util-decode-string": "^2.0.0", "unist-util-visit": "^5.0.0", "zwitch": "^2.0.0" } }, "sha512-xj68wMTvGXVOKonmog6LwyJKrYXZPvlwabaryTjLh9LuvovB/KAH+kvi8Gjj+7rJjsFi23nkUxRQv1KqSroMqA=="],
+
"mdast-util-mdx-jsx/parse-entities": ["[email protected]", "", { "dependencies": { "@types/unist": "^2.0.0", "character-entities-legacy": "^3.0.0", "character-reference-invalid": "^2.0.0", "decode-named-character-reference": "^1.0.0", "is-alphanumerical": "^2.0.0", "is-decimal": "^2.0.0", "is-hexadecimal": "^2.0.0" } }, "sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw=="],
+ "mdast-util-mdxjs-esm/mdast-util-to-markdown": ["[email protected]", "", { "dependencies": { "@types/mdast": "^4.0.0", "@types/unist": "^3.0.0", "longest-streak": "^3.0.0", "mdast-util-phrasing": "^4.0.0", "mdast-util-to-string": "^4.0.0", "micromark-util-classify-character": "^2.0.0", "micromark-util-decode-string": "^2.0.0", "unist-util-visit": "^5.0.0", "zwitch": "^2.0.0" } }, "sha512-xj68wMTvGXVOKonmog6LwyJKrYXZPvlwabaryTjLh9LuvovB/KAH+kvi8Gjj+7rJjsFi23nkUxRQv1KqSroMqA=="],
+
"parse-entities/character-entities": ["[email protected]", "", {}, "sha512-iBMyeEHxfVnIakwOuDXpVkc54HijNgCyQB2w0VfGQThle6NXn50zU6V/u+LDhxHcDUPojn6Kpga3PTAD8W1bQw=="],
"path-scurry/lru-cache": ["[email protected]", "", {}, "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ=="],
@@ -1281,6 +1310,8 @@
"refractor/prismjs": ["[email protected]", "", {}, "sha512-t13BGPUlFDR7wRB5kQDG4jjl7XeuH6jbJGt11JHPL96qwsEHNX2+68tFXqc1/k+/jALsbSWJKUOT/hcYAZ5LkA=="],
+ "remark-stringify/mdast-util-to-markdown": ["[email protected]", "", { "dependencies": { "@types/mdast": "^4.0.0", "@types/unist": "^3.0.0", "longest-streak": "^3.0.0", "mdast-util-phrasing": "^4.0.0", "mdast-util-to-string": "^4.0.0", "micromark-util-classify-character": "^2.0.0", "micromark-util-decode-string": "^2.0.0", "unist-util-visit": "^5.0.0", "zwitch": "^2.0.0" } }, "sha512-xj68wMTvGXVOKonmog6LwyJKrYXZPvlwabaryTjLh9LuvovB/KAH+kvi8Gjj+7rJjsFi23nkUxRQv1KqSroMqA=="],
+
"streamdown/remend": ["[email protected]", "", {}, "sha512-N3DiY5qbRPoa5vkxn1oDLMyXOVTeo6Hp+XOj6SIqJAYUgLS0Q587gILPMom/qm86AQ/ZrcOdwEIzCz8V3J0nxQ=="],
"streamdown/tailwind-merge": ["[email protected]", "", {}, "sha512-uxL7qAVQriqRQPAyK3pj66VqskWqoZ37PW94jwOTwNfq/z9oyu1V+eqrZqtR2+fCiXdYOZe/Modt8GtvqNzu+w=="],
diff --git a/webui/package-lock.json b/webui/package-lock.json
index a5314d770..0e64b7596 100644
--- a/webui/package-lock.json
+++ b/webui/package-lock.json
@@ -33,6 +33,7 @@
"react-syntax-highlighter": "^15.6.1",
"rehype-katex": "^7.0.1",
"remark-breaks": "^4.0.0",
+ "remark-cjk-friendly": "2.3.3",
"remark-gfm": "^4.0.0",
"remark-math": "^6.0.0",
"remark-parse": "^11.0.0",
@@ -4192,6 +4193,18 @@
"node": "6.* || 8.* || >= 10.*"
}
},
+ "node_modules/get-east-asian-width": {
+ "version": "1.7.0",
+ "resolved": "https://mirrors.tencent.com/npm/get-east-asian-width/-/get-east-asian-width-1.7.0.tgz",
+ "integrity": "sha512-XjH1AECxf0giL2V1aU8vKyRR2ppRUb5c0EvT7zuJTokQ74bNo52zOtghqdWIqrhUD79fo3x0WfKZdOqxF6LG1Q==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=18"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
+ },
"node_modules/get-nonce": {
"version": "1.0.1",
"license": "MIT",
@@ -5481,7 +5494,9 @@
}
},
"node_modules/mdast-util-to-markdown": {
- "version": "2.1.2",
+ "version": "2.2.0",
+ "resolved": "https://mirrors.tencent.com/npm/mdast-util-to-markdown/-/mdast-util-to-markdown-2.2.0.tgz",
+ "integrity": "sha512-2Jn/ADrHFFyyniBMi0bncTbJqU78K3/TEYvzvwrWWzD9MYPUa/Gm4r3nOeKFQg/PDM5BsbDYdc8ackW4NTY6NA==",
"license": "MIT",
"dependencies": {
"@types/mdast": "^4.0.0",
@@ -5489,8 +5504,10 @@
"longest-streak": "^3.0.0",
"mdast-util-phrasing": "^4.0.0",
"mdast-util-to-string": "^4.0.0",
+ "micromark-util-character": "^2.0.0",
"micromark-util-classify-character": "^2.0.0",
"micromark-util-decode-string": "^2.0.0",
+ "micromark-util-html-tag-name": "^2.0.0",
"unist-util-visit": "^5.0.0",
"zwitch": "^2.0.0"
},
@@ -5499,6 +5516,30 @@
"url": "https://opencollective.com/unified"
}
},
+ "node_modules/mdast-util-to-markdown-cjk-friendly": {
+ "version": "1.0.1",
+ "resolved": "https://mirrors.tencent.com/npm/mdast-util-to-markdown-cjk-friendly/-/mdast-util-to-markdown-cjk-friendly-1.0.1.tgz",
+ "integrity": "sha512-J3njpXAKUXGOa4k5pUwxHV0yuO4k5DgCrHGPady8xCFutjDr31UmuSaLuuxPLJNBO3336dX+CtB4GnNhgl0KKw==",
+ "license": "MIT",
+ "dependencies": {
+ "mdast-util-to-markdown": "^2.1.3",
+ "micromark-extension-cjk-friendly-util": "3.0.1",
+ "micromark-util-character": "^2.1.1",
+ "micromark-util-classify-character": "^2.0.1",
+ "micromark-util-html-tag-name": "^2.0.1"
+ },
+ "engines": {
+ "node": ">=18"
+ },
+ "peerDependencies": {
+ "@types/mdast": "*"
+ },
+ "peerDependenciesMeta": {
+ "@types/mdast": {
+ "optional": true
+ }
+ }
+ },
"node_modules/mdast-util-to-string": {
"version": "4.0.0",
"license": "MIT",
@@ -5589,6 +5630,50 @@
"micromark-util-types": "^2.0.0"
}
},
+ "node_modules/micromark-extension-cjk-friendly": {
+ "version": "2.0.2",
+ "resolved": "https://mirrors.tencent.com/npm/micromark-extension-cjk-friendly/-/micromark-extension-cjk-friendly-2.0.2.tgz",
+ "integrity": "sha512-/BqZNCkDWg3Hj9zEEcRitZ5uMF3X78qmBFuP17oAjnVDZ0+H1GHiDKMJZDpJsXasCHEn/kzl6QHozUiWgCHtPQ==",
+ "license": "MIT",
+ "dependencies": {
+ "devlop": "^1.1.0",
+ "micromark-extension-cjk-friendly-util": "3.0.1",
+ "micromark-util-chunked": "^2.0.1",
+ "micromark-util-resolve-all": "^2.0.1",
+ "micromark-util-symbol": "^2.0.1"
+ },
+ "engines": {
+ "node": ">=18"
+ },
+ "peerDependencies": {
+ "micromark": "^4.0.0",
+ "micromark-util-types": "^2.0.0"
+ },
+ "peerDependenciesMeta": {
+ "micromark-util-types": {
+ "optional": true
+ }
+ }
+ },
+ "node_modules/micromark-extension-cjk-friendly-util": {
+ "version": "3.0.1",
+ "resolved": "https://mirrors.tencent.com/npm/micromark-extension-cjk-friendly-util/-/micromark-extension-cjk-friendly-util-3.0.1.tgz",
+ "integrity": "sha512-GcbXqTTHOsiZHyF753oIddP/J2eH8j9zpyQPhkof6B2JNxfEJabnQqxbCgzJNuNes0Y2jTNJ3LiYPSXr6eJA8w==",
+ "license": "MIT",
+ "dependencies": {
+ "get-east-asian-width": "^1.4.0",
+ "micromark-util-character": "^2.1.1",
+ "micromark-util-symbol": "^2.0.1"
+ },
+ "engines": {
+ "node": ">=18"
+ },
+ "peerDependenciesMeta": {
+ "micromark-util-types": {
+ "optional": true
+ }
+ }
+ },
"node_modules/micromark-extension-gfm": {
"version": "3.0.0",
"license": "MIT",
@@ -6965,6 +7050,28 @@
"url": "https://opencollective.com/unified"
}
},
+ "node_modules/remark-cjk-friendly": {
+ "version": "2.3.3",
+ "resolved": "https://mirrors.tencent.com/npm/remark-cjk-friendly/-/remark-cjk-friendly-2.3.3.tgz",
+ "integrity": "sha512-tx+udvO1M3Pc36Mw4rXDWIFpeIqCzFdbmHq+IjteIEqnKkEjVACYR9FfUqCTDaKEjbjQQ1XlDq7tzRvuXw6Uzg==",
+ "license": "MIT",
+ "dependencies": {
+ "mdast-util-to-markdown-cjk-friendly": "1.0.1",
+ "micromark-extension-cjk-friendly": "2.0.2"
+ },
+ "engines": {
+ "node": ">=18"
+ },
+ "peerDependencies": {
+ "@types/mdast": "^4.0.0",
+ "unified": "^11.0.0"
+ },
+ "peerDependenciesMeta": {
+ "@types/mdast": {
+ "optional": true
+ }
+ }
+ },
"node_modules/remark-gfm": {
"version": "4.0.1",
"license": "MIT",
diff --git a/webui/package.json b/webui/package.json
index 8fd6eedce..869991bc0 100644
--- a/webui/package.json
+++ b/webui/package.json
@@ -43,6 +43,7 @@
"react-syntax-highlighter": "^15.6.1",
"rehype-katex": "^7.0.1",
"remark-breaks": "^4.0.0",
+ "remark-cjk-friendly": "2.3.3",
"remark-gfm": "^4.0.0",
"remark-math": "^6.0.0",
"remark-parse": "^11.0.0",
diff --git a/webui/src/components/MarkdownTextRenderer.tsx b/webui/src/components/MarkdownTextRenderer.tsx
index 9be380c4c..8fd369566 100644
--- a/webui/src/components/MarkdownTextRenderer.tsx
+++ b/webui/src/components/MarkdownTextRenderer.tsx
@@ -8,7 +8,7 @@ import {
} from "react";
import { useTranslation } from "react-i18next";
import { Check, Globe2 } from "lucide-react";
-import { decodeString } from "micromark-util-decode-string";
+import remarkCjkFriendly from "remark-cjk-friendly/parseOnly";
import remarkBreaks from "remark-breaks";
import remarkGfm from "remark-gfm";
import remarkMath from "remark-math";
@@ -61,10 +61,6 @@ type MarkdownAstNode = {
type: string;
value?: string;
children?: MarkdownAstNode[];
- position?: {
- start: { offset: number };
- end: { offset: number };
- };
data?: {
hName?: string;
};
@@ -263,54 +259,8 @@ function remarkSafeHtmlSubset() {
};
}
-// Model output can put CJK punctuation or whitespace before a closing `**`
-// followed immediately by a word, which CommonMark leaves as literal text.
-const STRONG_BEFORE_WORD = /(?<!\\)\*\*([^\s*][^*\r\n]*?)(?<!\\)\*\*(?=[\p{L}\p{N}])/gu;
-const CJK_CHARACTER = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u;
-
-function normalizeCjkStrongBoundaries(node: MarkdownAstNode, source: string): void {
- if (!node.children) return;
- node.children = node.children.flatMap((child) => {
- if (child.type !== "text" || !child.value?.includes("**") || !child.position) {
- normalizeCjkStrongBoundaries(child, source);
- return [child];
- }
-
- // Match source delimiters before decoding escapes and character references.
- // Paragraph continuation indentation is not part of the parsed text value.
- const textSource = source.slice(child.position.start.offset, child.position.end.offset)
- .replace(/(\r\n?|\n)[\t ]+/g, "$1");
-
- const replacement: MarkdownAstNode[] = [];
- let cursor = 0;
- for (const match of textSource.matchAll(STRONG_BEFORE_WORD)) {
- const start = match.index;
- const end = start + match[0].length;
- const content = decodeString(match[1]);
- if (!CJK_CHARACTER.test(content + textSource.slice(end, end + 1))) continue;
- const label = content.trimEnd();
- if (start > cursor) replacement.push(safeText(decodeString(textSource.slice(cursor, start))));
- replacement.push({
- type: "strong",
- children: [safeText(label)],
- });
- if (label.length < content.length) replacement.push(safeText(content.slice(label.length)));
- cursor = end;
- }
- if (cursor === 0) return [child];
- if (cursor < textSource.length) replacement.push(safeText(decodeString(textSource.slice(cursor))));
- return replacement;
- });
-}
-
-function remarkCjkStrongBoundaries() {
- return (tree: MarkdownAstNode, file: { toString(): string }) => {
- normalizeCjkStrongBoundaries(tree, file.toString());
- };
-}
-
const remarkPlugins: NonNullable<StreamdownProps["remarkPlugins"]> = [
- remarkCjkStrongBoundaries,
+ remarkCjkFriendly,
remarkBreaks,
[remarkGfm, { singleTilde: false }],
remarkCjkAutolinks,
diff --git a/webui/src/tests/markdown-text-renderer.test.tsx b/webui/src/tests/markdown-text-renderer.test.tsx
index 4aed21e7c..9b2833f8e 100644
--- a/webui/src/tests/markdown-text-renderer.test.tsx
+++ b/webui/src/tests/markdown-text-renderer.test.tsx
@@ -926,6 +926,23 @@ describe("MarkdownTextRenderer", () => {
expect(container.textContent).toBe(source);
});
+ it.each([
+ ["**`ThreadTranscript` 已经朝这个方向走了一步:它把正文、来源覆盖和分页接纳集中起来,因此可以看作一个隐式的状态转换模型。**但“集中到一个类”不等于已经有清晰的状态机;还要看合法状态、输入事件、转换条件和不变量是否明确。", "ThreadTranscript", "CODE"],
+ ["**参考 [说明](https://example.com) 的结论。**继续阅读。", "说明", "A"],
+ ])("parses CJK emphasis around nested inline syntax: %s", (source, label, tag) => {
+ const { container, rerender } = render(<MarkdownTextRenderer>{source}</MarkdownTextRenderer>);
+ expect(screen.getByText(label).tagName).toBe(tag);
+ expect(screen.getByText(label).closest("strong")).not.toBeNull();
+ if (tag === "A") expect(screen.getByText(label)).toHaveAttribute("href", "https://example.com");
+ expect(container.textContent).not.toContain("**");
+ for (let end = 1; end <= source.length; end += 1) {
+ rerender(<MarkdownTextRenderer streaming preserveStreamingLayout>{source.slice(0, end)}</MarkdownTextRenderer>);
+ }
+ expect(screen.getByText(label).closest("strong")).not.toBeNull();
+ rerender(<MarkdownTextRenderer preserveStreamingLayout>{source}</MarkdownTextRenderer>);
+ expect(screen.getByText(label).closest("strong")).not.toBeNull();
+ });
+
it("renders bold CJK text when more CJK text follows immediately", () => {
render(
<MarkdownTextRenderer streaming>
@@ -943,7 +960,7 @@ describe("MarkdownTextRenderer", () => {
it.each([
["**边界说明:**issue 只有截图,没有原始消息。", "issue 只有截图,没有原始消息。"],
- ["**边界说明: **issue 只有截图,没有原始消息。", " issue 只有截图,没有原始消息。"],
+ ["**边界说明:** issue 只有截图,没有原始消息。", " issue 只有截图,没有原始消息。"],
])("renders CJK bold labels before Latin text through stream completion: %s", (source, following) => {
const { container, rerender } = render(<MarkdownTextRenderer>{source}</MarkdownTextRenderer>);
expect(container.querySelector("strong")).toHaveTextContent("边界说明:");
@@ -958,6 +975,41 @@ describe("MarkdownTextRenderer", () => {
expect(container.textContent).toBe(`边界说明:${following}`);
});
+ it.each([
+ ["前文**「重点」**后文", "strong", "「重点」", "前文「重点」后文"],
+ ["**日本語。**続き", "strong", "日本語。", "日本語。続き"],
+ ["**한국어(설명)**입니다", "strong", "한국어(설명)", "한국어(설명)입니다"],
+ ["*说明。*继续", "em", "说明。", "说明。继续"],
+ ["**包含 *强调* 的结论。**继续", "strong em", "强调", "包含 强调 的结论。继续"],
+ ["**English.** Next", "strong", "English.", "English. Next"],
+ ["**中文。** 继续", "strong", "中文。", "中文。 继续"],
+ ["**中文**。继续", "strong", "中文", "中文。继续"],
+ ])("preserves emphasis grammar and CJK delimiter boundaries: %s", (source, selector, label, text) => {
+ const { container, rerender } = render(<MarkdownTextRenderer>{source}</MarkdownTextRenderer>);
+ expect(container.querySelector(selector)).toHaveTextContent(label);
+ expect(container.textContent).toBe(text);
+ rerender(<MarkdownTextRenderer streaming preserveStreamingLayout>{source}</MarkdownTextRenderer>);
+ expect(container.querySelector(selector)).toHaveTextContent(label);
+ expect(container.textContent).toBe(text);
+ rerender(<MarkdownTextRenderer preserveStreamingLayout>{source}</MarkdownTextRenderer>);
+ expect(container.querySelector(selector)).toHaveTextContent(label);
+ expect(container.textContent).toBe(text);
+ });
+
+ it.each([
+ "**边界说明: **issue 只有截图,没有原始消息。",
+ "** 边界说明:**继续说明。",
+ "**English.**Next",
+ ])("keeps invalid emphasis boundaries literal after streaming completes: %s", (source) => {
+ const { container, rerender } = render(<MarkdownTextRenderer>{source}</MarkdownTextRenderer>);
+ expect(container.querySelector("strong")).toBeNull();
+ expect(container.textContent).toBe(source);
+ rerender(<MarkdownTextRenderer streaming preserveStreamingLayout>{source}</MarkdownTextRenderer>);
+ rerender(<MarkdownTextRenderer preserveStreamingLayout>{source}</MarkdownTextRenderer>);
+ expect(container.querySelector("strong")).toBeNull();
+ expect(container.textContent).toBe(source);
+ });
+
it("keeps literal bold markers in code and escaped text", () => {
const { container } = render(
<MarkdownTextRenderer highlightCode={false}>
@@ -977,7 +1029,7 @@ describe("MarkdownTextRenderer", () => {
["\\*\\*字面符号:\\*\\*issue 和 **结论:**继续说明。", "**字面符号:**issue 和 结论:继续说明。"],
["&#42;&#42;字面符号:&#42;&#42;issue 和 **结论:**继续说明。", "**字面符号:**issue 和 结论:继续说明。"],
["- 第一行\n **结论:**继续 &amp; 其他说明。", "第一行\n结论:继续 & 其他说明。"],
- ])("recovers CJK bold labels without reinterpreting decoded text: %s", (source, text) => {
+ ])("parses CJK bold labels without reinterpreting decoded text: %s", (source, text) => {
const { container, rerender } = render(<MarkdownTextRenderer>{source}</MarkdownTextRenderer>);
expect(container.querySelectorAll("strong")).toHaveLength(1);
expect(container.textContent?.trim()).toBe(text);