World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
List as cited sources only the numbers the answer renders as citation marks (#952)
Maya Kid9e51bb1a8f6e
diff --git a/web/package.json b/web/package.jsonindex 968621bc..7a0e73fe 100644--- a/web/package.json+++ b/web/package.json@@ -33,10 +33,12 @@"react-markdown": "^10.1.0","rehype-highlight": "^7.0.2","remark-gfm": "^4.0.1",+ "remark-parse": "^11.0.0","remend": "^1.3.1","shadcn": "^4.21.0","sigma": "^3.0.3",- "tw-animate-css": "^1.4.0"+ "tw-animate-css": "^1.4.0",+ "unified": "^11.0.5"},"devDependencies": {"@tailwindcss/typography": "^0.5.20",diff --git a/web/pnpm-lock.yaml b/web/pnpm-lock.yamlindex e1a1e65f..2c2e451f 100644--- a/web/pnpm-lock.yaml+++ b/web/pnpm-lock.yaml@@ -68,6 +68,9 @@ importers:remark-gfm:specifier: ^4.0.1version: 4.0.1+ remark-parse:+ specifier: ^11.0.0+ version: 11.0.0remend:specifier: ^1.3.1version: 1.3.1@@ -80,6 +83,9 @@ importers:tw-animate-css:specifier: ^1.4.0version: 1.4.0+ unified:+ specifier: ^11.0.5+ version: 11.0.5devDependencies:'@tailwindcss/typography':specifier: ^0.5.20diff --git a/web/src/citations.ts b/web/src/citations.tsindex ebfaeada..ec72f4c4 100644--- a/web/src/citations.ts+++ b/web/src/citations.ts@@ -3,10 +3,17 @@// 模型写的是 `[1]`、`[1][2]`、`[1, 2]`、`[1,2]`。**哪些被画成角标,和哪些被列进// 落款,必须是同一套判定**——否则正文里点得动的那个号在下面找不到对应的行。// 所以这里只有一处形状,`liveAnswer.citedSources` 与 `rehypeCitations` 共用它。+// **判定也只有一处**:落款从前对整段原文跑正则,代码块里的 `[1, 2]`、链接+// `[1](…)` 都算进去了,而角标只画在正文文字上;`citedNumbers` 用渲染同一个解析器,+// 跳过同样的元素。//// 每次现造一个正则:`g` 的 `lastIndex` 跟着上一次调用走,共用一个实例会让// 第二段文本从中间开始匹配。+import remarkGfm from "remark-gfm";+import remarkParse from "remark-parse";+import { unified } from "unified";+/** `[1]` / `[1][2]` / `[1, 2]` / `[1,2]`——括号里只有数字与分隔符才算引用 */export const citeRe = () => /\[(\d+(?:\s*[,,]\s*\d+)*)\]/g;@@ -39,6 +46,47 @@ export function splitCitations(text: string): CitePiece[] {return out;}+/* ── 落款:正文里画成了角标的那些号 ──────────────────────────────────────── */++/** mdast 里我们读的那几个字段(同下面 hast 的理由:结构类型,不多一个依赖) */+type MdNode = { type: string; value?: string; children?: MdNode[] };++/** `OPAQUE`(a / code / pre)在 mdast 里的名字,外加原样 HTML:+ * react-markdown 把它当 raw 节点交给 rehype,`rehypeCitations` 不在里面画角标 */+const MD_OPAQUE = new Set(["link", "linkReference", "inlineCode", "code", "html"]);++/** 与渲染同一套 remark 解析(remark-gfm 管表格与自动链接) */+const markdown = unified().use(remarkParse).use(remarkGfm);++/** 按正文记忆:一轮对话里每来一个词元都要重算一遍,收了尾的回答文本不再变 */+const citedMemo = new Map<string, ReadonlySet<number>>();+const CITED_MEMO_LIMIT = 200;++/** 正文里会被画成角标的那些号。 */+export function citedNumbers(text: string): ReadonlySet<number> {+ const hit = citedMemo.get(text);+ if (hit) return hit;+ const cited = new Set<number>();+ collectCited(markdown.parse(text) as MdNode, cited);+ if (citedMemo.size >= CITED_MEMO_LIMIT) {+ const oldest = citedMemo.keys().next().value;+ if (oldest !== undefined) citedMemo.delete(oldest);+ }+ citedMemo.set(text, cited);+ return cited;+}++function collectCited(node: MdNode, into: Set<number>): void {+ if (node.type === "text" && typeof node.value === "string") {+ for (const piece of splitCitations(node.value)) {+ if ("cite" in piece) for (const n of piece.cite) into.add(n);+ }+ return;+ }+ if (MD_OPAQUE.has(node.type)) return;+ for (const child of node.children ?? []) collectCited(child, into);+}+/* ── rehype 插件 ────────────────────────────────────────────────────────── *//** hast 里我们要动的那两种节点。用结构类型而不是 `@types/hast`:diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.tsindex f2e1dc43..487419ae 100644--- a/web/src/i18n/en.ts+++ b/web/src/i18n/en.ts@@ -821,7 +821,7 @@ export const en = {deleteBtn: "Delete",cancel: "Cancel",// 这条回答背后一条来源都没有(#547)。是事实陈述,所以每条都挂,不猜哪条该挂- noSources: "No sources consulted",+ noSources: "No sources cited",// 预览浮窗右上角那条出路:看完这一段还想看整篇的人走这里openOriginal: "Open original",},diff --git a/web/src/liveAnswer.test.ts b/web/src/liveAnswer.test.tsindex b827a40e..67ca3f1f 100644--- a/web/src/liveAnswer.test.ts+++ b/web/src/liveAnswer.test.ts@@ -1,5 +1,10 @@+import { createElement } from "react";+import { renderToStaticMarkup } from "react-dom/server";+import Markdown from "react-markdown";+import remarkGfm from "remark-gfm";import { describe, expect, it } from "vitest";import type { Source } from "./api";+import { rehypeCitations } from "./citations";import { answeredWithoutSources, citedSources, type Turn } from "./liveAnswer";const answer = (over: Partial<Turn> = {}): Turn => ({@@ -32,6 +37,44 @@ describe("citedSources", () => {});});+/** 正文渲染出来的角标:与 Chat 页同一组插件 */+function marks(text: string): number[] {+ const html = renderToStaticMarkup(+ createElement(Markdown, { remarkPlugins: [remarkGfm], rehypePlugins: [rehypeCitations] }, text),+ );+ const out = new Set<number>();+ for (const m of html.matchAll(/href="#cite-([\d,]+)"/g)) {+ for (const n of m[1].split(",")) out.add(Number(n));+ }+ return [...out].sort((a, b) => a - b);+}++describe("citedSources lists what the answer shows as a citation mark", () => {+ const at = (content: string) => citedSources(answer({ content, sources: six })).map((s) => s.n);++ it("skips numbers in code blocks, inline code and link text", () => {+ expect(at("```json\n[4, 5]\n```\nThe figure is in [1].")).toEqual([1]);+ expect(at("Inline `rows[2]` and `[3]` are code; [6] is a mark.")).toEqual([6]);+ expect(at("A link [4](https://example.com) is not a mark; [5] is.")).toEqual([5]);+ });++ it("counts an escaped bracket, which renders as a mark", () => {+ expect(at("An escaped \\[2\\] still reads as a mark.")).toEqual([2]);+ });++ it.each([+ "Revenue grew [1]; see also [2, 3].",+ "```json\n[4, 5]\n```\nThe figure is in [1].",+ "Inline `rows[2]` and `[3]` are code; [6] is a mark.",+ "A link [4](https://example.com) is not a mark; [5] is.",+ "An escaped \\[2\\] still reads as a mark.",+ "| metric | source |\n|---|---|\n| revenue | [3] |",+ "> quoted [1]\n\n- item [2]\n- item [4]",+ ])("agrees with the marks the answer renders: %j", (content) => {+ expect(at(content)).toEqual(marks(content));+ });+});+describe("answeredWithoutSources (#547)", () => {it("marks a finished answer with no sources", () => {expect(answeredWithoutSources(answer({ sources: [] }), false)).toBe(true);diff --git a/web/src/liveAnswer.ts b/web/src/liveAnswer.tsindex 40a14b24..de8c2717 100644--- a/web/src/liveAnswer.ts+++ b/web/src/liveAnswer.ts@@ -19,7 +19,7 @@// 于是改成一张表:谁开场谁拿句柄,读谁写谁都有名有姓。`send` 的守卫不用改——// 它本来问的就是「这一场在不在流」,现在这个问题终于只关于这一场。import type { ChatStep, Source } from "./api";-import { citeNumbers, citeRe } from "./citations";+import { citedNumbers } from "./citations";export interface Turn {role: "user" | "assistant";@@ -33,14 +33,11 @@ export interface Turn {** `sources` 是这一轮**检索到**的全部,不是回答**用到**的:打个招呼也可能顺手搜了* 一次,六条摘录挂在「你好」下面,读起来像是这句问候有六个出处。所以只列正文里- * 出现过 `[n]` 的那几条,编号照原样不重排,与正文里的标记对得上。- * `[1][2]`、`[1, 2]`、`[1,2]` 都认 */+ * 画成了角标的那几条,编号照原样不重排,与正文里的标记对得上。+ * `[1][2]`、`[1, 2]`、`[1,2]` 都认;代码与链接里的方括号不算,正文里也不画 */export function citedSources(turn: Turn): Source[] {if (!turn.sources?.length) return [];- const cited = new Set<number>();- for (const m of turn.content.matchAll(citeRe())) {- for (const n of citeNumbers(m[1])) cited.add(n);- }+ const cited = citedNumbers(turn.content);return turn.sources.filter((s) => cited.has(s.n));}