Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(modelcontext): keep literal ref text out of citation expansion (#4748)
AsanoReo82aeba357aa8
diff --git a/internal/modelcontext/citations.go b/internal/modelcontext/citations.goindex 53b391aa0..66dfc6dee 100644--- a/internal/modelcontext/citations.go+++ b/internal/modelcontext/citations.go@@ -199,7 +199,8 @@ func (r *sourceRegistry) registerLabeledReferences(text string) {}// ExpandText converts the private model protocol into the existing public-// <kb/> / <web/> contract. Unknown handles fail closed and disappear.+// <kb/> / <web/> contract. Unknown source handles fail closed and disappear.+// A <ref> tag that does not name a source handle is literal text and stays.func (r *sourceRegistry) ExpandText(text string) string {if r == nil || text == "" {return text@@ -208,16 +209,9 @@ func (r *sourceRegistry) ExpandText(text string) string {// by the model, then create canonical tags solely from registered handles.text = modelKBTagRE.ReplaceAllString(text, "")text = modelWebTagRE.ReplaceAllString(text, "")- if !r.citationsEnabled {- return refCandidateRE.ReplaceAllString(text, "")- }- return refCandidateRE.ReplaceAllStringFunc(text, withJSONCitationEscaping(func(tag string) string {- match := refTagRE.FindStringSubmatch(tag)- if len(match) != 2 {- return ""- }- return r.citationTag(match[1])- }))+ return refCandidateRE.ReplaceAllStringFunc(text, func(tag string) string {+ return rewriteRefTag(r, tag)+ })}// citationTag renders the public citation for a handle backed by current@@ -283,7 +277,7 @@ func (d *citationStreamExpander) Feed(chunk string) string {break}tag := data[:end+1]- out.WriteString(d.registry.ExpandText(tag))+ out.WriteString(rewriteRefTag(d.registry, tag))data = data[end+1:]continue}@@ -334,9 +328,63 @@ func (d *citationStreamExpander) Flush() string {}pending := d.pendingd.pending = ""- lower := strings.ToLower(pending)- if isSourceTagPending(lower) {+ if isSourceTagPending(strings.ToLower(pending)) && !keepLiteralRef(pending) {return ""}- return d.registry.ExpandText(pending)+ return rewriteRefTag(d.registry, pending)+}++// rewriteRefTag expands one citation tag. Tags that do not name a source+// handle are returned unchanged so a quoted syntax example survives.+func rewriteRefTag(r *sourceRegistry, tag string) string {+ return withJSONCitationEscaping(func(decoded string) string {+ if keepLiteralRef(decoded) {+ return decoded+ }+ if r == nil || !r.citationsEnabled {+ return ""+ }+ match := refTagRE.FindStringSubmatch(decoded)+ if len(match) != 2 {+ return ""+ }+ return r.citationTag(match[1])+ })(tag)+}++var (+ refIDAttrRE = regexp.MustCompile(`(?i)\bid\s*=\s*"([^"]*)"`)+ refIDAttrSingleRE = regexp.MustCompile(`(?i)\bid\s*=\s*'([^']*)'`)+)++// keepLiteralRef reports a <ref> tag that is not a citation. c1/w1 are the+// legacy spellings of chunk://1 and url://1, so they stay on the citation path.+func keepLiteralRef(tag string) bool {+ if !strings.HasPrefix(strings.ToLower(tag), "<ref") {+ return false+ }+ // A scheme inside the tag is a citation attempt, even when the markup is+ // nested or only partly decoded. Leaving it in place lets the handle pass+ // rewrite the token and break the surrounding text.+ if strings.Contains(tag, "://") {+ return false+ }+ id, ok := citedRefID(tag)+ if !ok {+ return true+ }+ if id == "" || canonicalSourceHandle(id) != "" {+ return false+ }+ return true+}++func citedRefID(tag string) (string, bool) {+ if match := refIDAttrRE.FindStringSubmatch(tag); len(match) == 2 {+ return html.UnescapeString(match[1]), true+ }+ if match := refIDAttrSingleRE.FindStringSubmatch(tag); len(match) == 2 {+ return html.UnescapeString(match[1]), true+ }+ return "", false}diff --git a/internal/modelcontext/citations_literal_test.go b/internal/modelcontext/citations_literal_test.gonew file mode 100644index 000000000..61cdf7e41--- /dev/null+++ b/internal/modelcontext/citations_literal_test.go@@ -0,0 +1,116 @@+package modelcontext++import (+ "strings"+ "testing"++ "github.com/stretchr/testify/require"+)++func TestLiteralRefTextSurvivesCitationExpansion(t *testing.T) {+ cases := []struct {+ name string+ register bool+ in string+ want string+ }{+ {+ name: "unregistered legacy id stays non-citable",+ in: `引用标签的写法是 <ref id="c1"/>。`,+ want: `引用标签的写法是 。`,+ },+ {+ name: "registered legacy id still expands",+ register: true,+ in: `引用标签的写法是 <ref id="c1"/>。`,+ want: `引用标签的写法是 <kb doc="Doc" chunk_id="chunk-1" />。`,+ },+ {+ name: "weknora attribute",+ in: `引用标签的写法是 <ref weknora_id="c1"/>。`,+ want: `引用标签的写法是 <ref weknora_id="c1"/>。`,+ },+ {+ name: "arbitrary text",+ in: `引用标签的写法是 <ref arbitrary text>。`,+ want: `引用标签的写法是 <ref arbitrary text>。`,+ },+ {+ name: "inline code keeps a non-handle tag",+ register: true,+ in: "写法是 `<ref id=\"example\"/>`。",+ want: "写法是 `<ref id=\"example\"/>`。",+ },+ {+ name: "fenced code keeps a literal tag",+ register: true,+ in: "示例:\n```\n<ref weknora_id=\"c1\"/>\n```\n结束",+ want: "示例:\n```\n<ref weknora_id=\"c1\"/>\n```\n结束",+ },+ {+ name: "genuine citation still expands",+ register: true,+ in: `依据 <ref id="chunk://1"/>。`,+ want: `依据 <kb doc="Doc" chunk_id="chunk-1" />。`,+ },+ {+ name: "fabricated handle is still removed",+ in: `伪造 <ref id="chunk://999"/>。`,+ want: `伪造 。`,+ },+ {+ name: "document handle is still removed",+ in: `文档 <ref id="doc://1"/>。`,+ want: `文档 。`,+ },+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ registry := newSourceRegistry()+ if tc.register {+ registry.RegisterChunk(ChunkReference{ChunkID: "chunk-1", DocumentTitle: "Doc"})+ registry.RegisterWeb("https://example.com", "Example")+ }+ require.Equal(t, tc.want, registry.ExpandText(tc.in))+ var streamed strings.Builder+ decoder := newCitationStreamExpander(registry)+ for i := range len(tc.in) {+ streamed.WriteString(decoder.Feed(tc.in[i : i+1]))+ }+ streamed.WriteString(decoder.Flush())+ require.Equal(t, tc.want, streamed.String())+ })+ }+}++func TestLiteralRefSplitAcrossChunks(t *testing.T) {+ registry := newSourceRegistry()+ text := `写法是 <ref weknora_id="c1"/>,以及 <ref id="example"/>。`+ for split := 0; split <= len(text); split++ {+ decoder := newCitationStreamExpander(registry)+ got := decoder.Feed(text[:split]) + decoder.Feed(text[split:]) + decoder.Flush()+ require.Equal(t, text, got, "split at %d", split)+ }+}++func TestFencedRefSplitAcrossChunksStaysLiteral(t *testing.T) {+ registry := newSourceRegistry()+ registry.RegisterChunk(ChunkReference{ChunkID: "chunk-1", DocumentTitle: "Doc"})+ text := "```html\n<ref id=\"example\"/>\n```"+ for split := 0; split <= len(text); split++ {+ decoder := newCitationStreamExpander(registry)+ got := decoder.Feed(text[:split]) + decoder.Feed(text[split:]) + decoder.Flush()+ require.Equal(t, text, got, "split at %d", split)+ }+}++func TestDisabledCitationsStillDropProtocolRefs(t *testing.T) {+ registry := newSourceRegistry(false)+ registry.RegisterChunk(ChunkReference{ChunkID: "chunk-1", DocumentTitle: "Doc"})+ require.Equal(t, "literal <ref weknora_id=\"c1\"/> done",+ registry.ExpandText("literal <ref weknora_id=\"c1\"/> done"))+ require.Equal(t, "legacy done",+ registry.ExpandText("legacy <ref id=\"c1\"/> done"))+ require.Equal(t, "protocol done",+ registry.ExpandText("protocol <ref id=\"chunk://1\"/> done"))+}