Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(modelcontext): keep literal ref text out of citation expansion (#4748)

AsanoReo82aeba357aa8
diff --git a/internal/modelcontext/citations.go b/internal/modelcontext/citations.go
index 53b391aa0..66dfc6dee 100644
--- a/internal/modelcontext/citations.go
+++ b/internal/modelcontext/citations.go
@@ -199,7 +199,8 @@ func (r *sourceRegistry) registerLabeledReferences(text string) {
}
// ExpandText converts the private model protocol into the existing public
-// <kb/> / <web/> contract. Unknown handles fail closed and disappear.
+// <kb/> / <web/> contract. Unknown source handles fail closed and disappear.
+// A <ref> tag that does not name a source handle is literal text and stays.
func (r *sourceRegistry) ExpandText(text string) string {
if r == nil || text == "" {
return text
@@ -208,16 +209,9 @@ func (r *sourceRegistry) ExpandText(text string) string {
// by the model, then create canonical tags solely from registered handles.
text = modelKBTagRE.ReplaceAllString(text, "")
text = modelWebTagRE.ReplaceAllString(text, "")
- if !r.citationsEnabled {
- return refCandidateRE.ReplaceAllString(text, "")
- }
- return refCandidateRE.ReplaceAllStringFunc(text, withJSONCitationEscaping(func(tag string) string {
- match := refTagRE.FindStringSubmatch(tag)
- if len(match) != 2 {
- return ""
- }
- return r.citationTag(match[1])
- }))
+ return refCandidateRE.ReplaceAllStringFunc(text, func(tag string) string {
+ return rewriteRefTag(r, tag)
+ })
}
// citationTag renders the public citation for a handle backed by current
@@ -283,7 +277,7 @@ func (d *citationStreamExpander) Feed(chunk string) string {
break
}
tag := data[:end+1]
- out.WriteString(d.registry.ExpandText(tag))
+ out.WriteString(rewriteRefTag(d.registry, tag))
data = data[end+1:]
continue
}
@@ -334,9 +328,63 @@ func (d *citationStreamExpander) Flush() string {
}
pending := d.pending
d.pending = ""
- lower := strings.ToLower(pending)
- if isSourceTagPending(lower) {
+ if isSourceTagPending(strings.ToLower(pending)) && !keepLiteralRef(pending) {
return ""
}
- return d.registry.ExpandText(pending)
+ return rewriteRefTag(d.registry, pending)
+}
+
+// rewriteRefTag expands one citation tag. Tags that do not name a source
+// handle are returned unchanged so a quoted syntax example survives.
+func rewriteRefTag(r *sourceRegistry, tag string) string {
+ return withJSONCitationEscaping(func(decoded string) string {
+ if keepLiteralRef(decoded) {
+ return decoded
+ }
+ if r == nil || !r.citationsEnabled {
+ return ""
+ }
+ match := refTagRE.FindStringSubmatch(decoded)
+ if len(match) != 2 {
+ return ""
+ }
+ return r.citationTag(match[1])
+ })(tag)
+}
+
+var (
+ refIDAttrRE = regexp.MustCompile(`(?i)\bid\s*=\s*"([^"]*)"`)
+ refIDAttrSingleRE = regexp.MustCompile(`(?i)\bid\s*=\s*'([^']*)'`)
+)
+
+// keepLiteralRef reports a <ref> tag that is not a citation. c1/w1 are the
+// legacy spellings of chunk://1 and url://1, so they stay on the citation path.
+func keepLiteralRef(tag string) bool {
+ if !strings.HasPrefix(strings.ToLower(tag), "<ref") {
+ return false
+ }
+ // A scheme inside the tag is a citation attempt, even when the markup is
+ // nested or only partly decoded. Leaving it in place lets the handle pass
+ // rewrite the token and break the surrounding text.
+ if strings.Contains(tag, "://") {
+ return false
+ }
+ id, ok := citedRefID(tag)
+ if !ok {
+ return true
+ }
+ if id == "" || canonicalSourceHandle(id) != "" {
+ return false
+ }
+ return true
+}
+
+func citedRefID(tag string) (string, bool) {
+ if match := refIDAttrRE.FindStringSubmatch(tag); len(match) == 2 {
+ return html.UnescapeString(match[1]), true
+ }
+ if match := refIDAttrSingleRE.FindStringSubmatch(tag); len(match) == 2 {
+ return html.UnescapeString(match[1]), true
+ }
+ return "", false
}
diff --git a/internal/modelcontext/citations_literal_test.go b/internal/modelcontext/citations_literal_test.go
new file mode 100644
index 000000000..61cdf7e41
--- /dev/null
+++ b/internal/modelcontext/citations_literal_test.go
@@ -0,0 +1,116 @@
+package modelcontext
+
+import (
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestLiteralRefTextSurvivesCitationExpansion(t *testing.T) {
+ cases := []struct {
+ name string
+ register bool
+ in string
+ want string
+ }{
+ {
+ name: "unregistered legacy id stays non-citable",
+ in: `引用标签的写法是 <ref id="c1"/>。`,
+ want: `引用标签的写法是 。`,
+ },
+ {
+ name: "registered legacy id still expands",
+ register: true,
+ in: `引用标签的写法是 <ref id="c1"/>。`,
+ want: `引用标签的写法是 <kb doc="Doc" chunk_id="chunk-1" />。`,
+ },
+ {
+ name: "weknora attribute",
+ in: `引用标签的写法是 <ref weknora_id="c1"/>。`,
+ want: `引用标签的写法是 <ref weknora_id="c1"/>。`,
+ },
+ {
+ name: "arbitrary text",
+ in: `引用标签的写法是 <ref arbitrary text>。`,
+ want: `引用标签的写法是 <ref arbitrary text>。`,
+ },
+ {
+ name: "inline code keeps a non-handle tag",
+ register: true,
+ in: "写法是 `<ref id=\"example\"/>`。",
+ want: "写法是 `<ref id=\"example\"/>`。",
+ },
+ {
+ name: "fenced code keeps a literal tag",
+ register: true,
+ in: "示例:\n```\n<ref weknora_id=\"c1\"/>\n```\n结束",
+ want: "示例:\n```\n<ref weknora_id=\"c1\"/>\n```\n结束",
+ },
+ {
+ name: "genuine citation still expands",
+ register: true,
+ in: `依据 <ref id="chunk://1"/>。`,
+ want: `依据 <kb doc="Doc" chunk_id="chunk-1" />。`,
+ },
+ {
+ name: "fabricated handle is still removed",
+ in: `伪造 <ref id="chunk://999"/>。`,
+ want: `伪造 。`,
+ },
+ {
+ name: "document handle is still removed",
+ in: `文档 <ref id="doc://1"/>。`,
+ want: `文档 。`,
+ },
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ registry := newSourceRegistry()
+ if tc.register {
+ registry.RegisterChunk(ChunkReference{ChunkID: "chunk-1", DocumentTitle: "Doc"})
+ registry.RegisterWeb("https://example.com", "Example")
+ }
+ require.Equal(t, tc.want, registry.ExpandText(tc.in))
+ var streamed strings.Builder
+ decoder := newCitationStreamExpander(registry)
+ for i := range len(tc.in) {
+ streamed.WriteString(decoder.Feed(tc.in[i : i+1]))
+ }
+ streamed.WriteString(decoder.Flush())
+ require.Equal(t, tc.want, streamed.String())
+ })
+ }
+}
+
+func TestLiteralRefSplitAcrossChunks(t *testing.T) {
+ registry := newSourceRegistry()
+ text := `写法是 <ref weknora_id="c1"/>,以及 <ref id="example"/>。`
+ for split := 0; split <= len(text); split++ {
+ decoder := newCitationStreamExpander(registry)
+ got := decoder.Feed(text[:split]) + decoder.Feed(text[split:]) + decoder.Flush()
+ require.Equal(t, text, got, "split at %d", split)
+ }
+}
+
+func TestFencedRefSplitAcrossChunksStaysLiteral(t *testing.T) {
+ registry := newSourceRegistry()
+ registry.RegisterChunk(ChunkReference{ChunkID: "chunk-1", DocumentTitle: "Doc"})
+ text := "```html\n<ref id=\"example\"/>\n```"
+ for split := 0; split <= len(text); split++ {
+ decoder := newCitationStreamExpander(registry)
+ got := decoder.Feed(text[:split]) + decoder.Feed(text[split:]) + decoder.Flush()
+ require.Equal(t, text, got, "split at %d", split)
+ }
+}
+
+func TestDisabledCitationsStillDropProtocolRefs(t *testing.T) {
+ registry := newSourceRegistry(false)
+ registry.RegisterChunk(ChunkReference{ChunkID: "chunk-1", DocumentTitle: "Doc"})
+ require.Equal(t, "literal <ref weknora_id=\"c1\"/> done",
+ registry.ExpandText("literal <ref weknora_id=\"c1\"/> done"))
+ require.Equal(t, "legacy done",
+ registry.ExpandText("legacy <ref id=\"c1\"/> done"))
+ require.Equal(t, "protocol done",
+ registry.ExpandText("protocol <ref id=\"chunk://1\"/> done"))
+}