Skip to content

World's first open-source enterprise world model.

List as cited sources only the numbers the answer renders as citation marks (#952)

Maya Kid9e51bb1a8f6e
diff --git a/web/package.json b/web/package.json
index 968621bc..7a0e73fe 100644
--- a/web/package.json
+++ b/web/package.json
@@ -33,10 +33,12 @@
"react-markdown": "^10.1.0",
"rehype-highlight": "^7.0.2",
"remark-gfm": "^4.0.1",
+ "remark-parse": "^11.0.0",
"remend": "^1.3.1",
"shadcn": "^4.21.0",
"sigma": "^3.0.3",
- "tw-animate-css": "^1.4.0"
+ "tw-animate-css": "^1.4.0",
+ "unified": "^11.0.5"
},
"devDependencies": {
"@tailwindcss/typography": "^0.5.20",
diff --git a/web/pnpm-lock.yaml b/web/pnpm-lock.yaml
index e1a1e65f..2c2e451f 100644
--- a/web/pnpm-lock.yaml
+++ b/web/pnpm-lock.yaml
@@ -68,6 +68,9 @@ importers:
remark-gfm:
specifier: ^4.0.1
version: 4.0.1
+ remark-parse:
+ specifier: ^11.0.0
+ version: 11.0.0
remend:
specifier: ^1.3.1
version: 1.3.1
@@ -80,6 +83,9 @@ importers:
tw-animate-css:
specifier: ^1.4.0
version: 1.4.0
+ unified:
+ specifier: ^11.0.5
+ version: 11.0.5
devDependencies:
'@tailwindcss/typography':
specifier: ^0.5.20
diff --git a/web/src/citations.ts b/web/src/citations.ts
index ebfaeada..ec72f4c4 100644
--- a/web/src/citations.ts
+++ b/web/src/citations.ts
@@ -3,10 +3,17 @@
// 模型写的是 `[1]`、`[1][2]`、`[1, 2]`、`[1,2]`。**哪些被画成角标,和哪些被列进
// 落款,必须是同一套判定**——否则正文里点得动的那个号在下面找不到对应的行。
// 所以这里只有一处形状,`liveAnswer.citedSources` 与 `rehypeCitations` 共用它。
+// **判定也只有一处**:落款从前对整段原文跑正则,代码块里的 `[1, 2]`、链接
+// `[1](…)` 都算进去了,而角标只画在正文文字上;`citedNumbers` 用渲染同一个解析器,
+// 跳过同样的元素。
//
// 每次现造一个正则:`g` 的 `lastIndex` 跟着上一次调用走,共用一个实例会让
// 第二段文本从中间开始匹配。
+import remarkGfm from "remark-gfm";
+import remarkParse from "remark-parse";
+import { unified } from "unified";
+
/** `[1]` / `[1][2]` / `[1, 2]` / `[1,2]`——括号里只有数字与分隔符才算引用 */
export const citeRe = () => /\[(\d+(?:\s*[,,]\s*\d+)*)\]/g;
@@ -39,6 +46,47 @@ export function splitCitations(text: string): CitePiece[] {
return out;
}
+/* ── 落款:正文里画成了角标的那些号 ──────────────────────────────────────── */
+
+/** mdast 里我们读的那几个字段(同下面 hast 的理由:结构类型,不多一个依赖) */
+type MdNode = { type: string; value?: string; children?: MdNode[] };
+
+/** `OPAQUE`(a / code / pre)在 mdast 里的名字,外加原样 HTML:
+ * react-markdown 把它当 raw 节点交给 rehype,`rehypeCitations` 不在里面画角标 */
+const MD_OPAQUE = new Set(["link", "linkReference", "inlineCode", "code", "html"]);
+
+/** 与渲染同一套 remark 解析(remark-gfm 管表格与自动链接) */
+const markdown = unified().use(remarkParse).use(remarkGfm);
+
+/** 按正文记忆:一轮对话里每来一个词元都要重算一遍,收了尾的回答文本不再变 */
+const citedMemo = new Map<string, ReadonlySet<number>>();
+const CITED_MEMO_LIMIT = 200;
+
+/** 正文里会被画成角标的那些号。 */
+export function citedNumbers(text: string): ReadonlySet<number> {
+ const hit = citedMemo.get(text);
+ if (hit) return hit;
+ const cited = new Set<number>();
+ collectCited(markdown.parse(text) as MdNode, cited);
+ if (citedMemo.size >= CITED_MEMO_LIMIT) {
+ const oldest = citedMemo.keys().next().value;
+ if (oldest !== undefined) citedMemo.delete(oldest);
+ }
+ citedMemo.set(text, cited);
+ return cited;
+}
+
+function collectCited(node: MdNode, into: Set<number>): void {
+ if (node.type === "text" && typeof node.value === "string") {
+ for (const piece of splitCitations(node.value)) {
+ if ("cite" in piece) for (const n of piece.cite) into.add(n);
+ }
+ return;
+ }
+ if (MD_OPAQUE.has(node.type)) return;
+ for (const child of node.children ?? []) collectCited(child, into);
+}
+
/* ── rehype 插件 ────────────────────────────────────────────────────────── */
/** hast 里我们要动的那两种节点。用结构类型而不是 `@types/hast`:
diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts
index f2e1dc43..487419ae 100644
--- a/web/src/i18n/en.ts
+++ b/web/src/i18n/en.ts
@@ -821,7 +821,7 @@ export const en = {
deleteBtn: "Delete",
cancel: "Cancel",
// 这条回答背后一条来源都没有(#547)。是事实陈述,所以每条都挂,不猜哪条该挂
- noSources: "No sources consulted",
+ noSources: "No sources cited",
// 预览浮窗右上角那条出路:看完这一段还想看整篇的人走这里
openOriginal: "Open original",
},
diff --git a/web/src/liveAnswer.test.ts b/web/src/liveAnswer.test.ts
index b827a40e..67ca3f1f 100644
--- a/web/src/liveAnswer.test.ts
+++ b/web/src/liveAnswer.test.ts
@@ -1,5 +1,10 @@
+import { createElement } from "react";
+import { renderToStaticMarkup } from "react-dom/server";
+import Markdown from "react-markdown";
+import remarkGfm from "remark-gfm";
import { describe, expect, it } from "vitest";
import type { Source } from "./api";
+import { rehypeCitations } from "./citations";
import { answeredWithoutSources, citedSources, type Turn } from "./liveAnswer";
const answer = (over: Partial<Turn> = {}): Turn => ({
@@ -32,6 +37,44 @@ describe("citedSources", () => {
});
});
+/** 正文渲染出来的角标:与 Chat 页同一组插件 */
+function marks(text: string): number[] {
+ const html = renderToStaticMarkup(
+ createElement(Markdown, { remarkPlugins: [remarkGfm], rehypePlugins: [rehypeCitations] }, text),
+ );
+ const out = new Set<number>();
+ for (const m of html.matchAll(/href="#cite-([\d,]+)"/g)) {
+ for (const n of m[1].split(",")) out.add(Number(n));
+ }
+ return [...out].sort((a, b) => a - b);
+}
+
+describe("citedSources lists what the answer shows as a citation mark", () => {
+ const at = (content: string) => citedSources(answer({ content, sources: six })).map((s) => s.n);
+
+ it("skips numbers in code blocks, inline code and link text", () => {
+ expect(at("```json\n[4, 5]\n```\nThe figure is in [1].")).toEqual([1]);
+ expect(at("Inline `rows[2]` and `[3]` are code; [6] is a mark.")).toEqual([6]);
+ expect(at("A link [4](https://example.com) is not a mark; [5] is.")).toEqual([5]);
+ });
+
+ it("counts an escaped bracket, which renders as a mark", () => {
+ expect(at("An escaped \\[2\\] still reads as a mark.")).toEqual([2]);
+ });
+
+ it.each([
+ "Revenue grew [1]; see also [2, 3].",
+ "```json\n[4, 5]\n```\nThe figure is in [1].",
+ "Inline `rows[2]` and `[3]` are code; [6] is a mark.",
+ "A link [4](https://example.com) is not a mark; [5] is.",
+ "An escaped \\[2\\] still reads as a mark.",
+ "| metric | source |\n|---|---|\n| revenue | [3] |",
+ "> quoted [1]\n\n- item [2]\n- item [4]",
+ ])("agrees with the marks the answer renders: %j", (content) => {
+ expect(at(content)).toEqual(marks(content));
+ });
+});
+
describe("answeredWithoutSources (#547)", () => {
it("marks a finished answer with no sources", () => {
expect(answeredWithoutSources(answer({ sources: [] }), false)).toBe(true);
diff --git a/web/src/liveAnswer.ts b/web/src/liveAnswer.ts
index 40a14b24..de8c2717 100644
--- a/web/src/liveAnswer.ts
+++ b/web/src/liveAnswer.ts
@@ -19,7 +19,7 @@
// 于是改成一张表:谁开场谁拿句柄,读谁写谁都有名有姓。`send` 的守卫不用改——
// 它本来问的就是「这一场在不在流」,现在这个问题终于只关于这一场。
import type { ChatStep, Source } from "./api";
-import { citeNumbers, citeRe } from "./citations";
+import { citedNumbers } from "./citations";
export interface Turn {
role: "user" | "assistant";
@@ -33,14 +33,11 @@ export interface Turn {
*
* `sources` 是这一轮**检索到**的全部,不是回答**用到**的:打个招呼也可能顺手搜了
* 一次,六条摘录挂在「你好」下面,读起来像是这句问候有六个出处。所以只列正文里
- * 出现过 `[n]` 的那几条,编号照原样不重排,与正文里的标记对得上。
- * `[1][2]`、`[1, 2]`、`[1,2]` 都认 */
+ * 画成了角标的那几条,编号照原样不重排,与正文里的标记对得上。
+ * `[1][2]`、`[1, 2]`、`[1,2]` 都认;代码与链接里的方括号不算,正文里也不画 */
export function citedSources(turn: Turn): Source[] {
if (!turn.sources?.length) return [];
- const cited = new Set<number>();
- for (const m of turn.content.matchAll(citeRe())) {
- for (const n of citeNumbers(m[1])) cited.add(n);
- }
+ const cited = citedNumbers(turn.content);
return turn.sources.filter((s) => cited.has(s.n));
}