Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(modelcontext): preserve citations in JSON strings (#4733)

errpunkdea0a30c5f23
diff --git a/internal/modelcontext/citations.go b/internal/modelcontext/citations.go
index e0e3770ab..53b391aa0 100644
--- a/internal/modelcontext/citations.go
+++ b/internal/modelcontext/citations.go
@@ -5,6 +5,7 @@
package modelcontext
import (
+ "encoding/json"
"fmt"
"html"
"regexp"
@@ -109,7 +110,7 @@ func (r *sourceRegistry) CompactPublicCitations(text string, evidence bool) stri
if r == nil || text == "" {
return text
}
- text = publicKBTagRE.ReplaceAllStringFunc(text, func(tag string) string {
+ text = publicKBTagRE.ReplaceAllStringFunc(text, withJSONCitationEscaping(func(tag string) string {
chunkID := publicAttr(chunkAttrRE, tag)
if chunkID == "" {
return tag
@@ -120,15 +121,34 @@ func (r *sourceRegistry) CompactPublicCitations(text string, evidence bool) stri
DocumentTitle: publicAttr(docAttrRE, tag),
}, evidence)
return `<ref id="` + handle + `"/>`
- })
- return publicWebTagRE.ReplaceAllStringFunc(text, func(tag string) string {
+ }))
+ return publicWebTagRE.ReplaceAllStringFunc(text, withJSONCitationEscaping(func(tag string) string {
rawURL := publicAttr(urlAttrRE, tag)
if rawURL == "" {
return tag
}
handle := r.registerWeb(rawURL, publicAttr(titleAttrRE, tag), evidence)
return `<ref id="` + handle + `"/>`
- })
+ }))
+}
+
+// withJSONCitationEscaping preserves one layer of JSON string escaping around a citation.
+func withJSONCitationEscaping(rewrite func(string) string) func(string) string {
+ return func(tag string) string {
+ if !jsonCitationTagRE.MatchString(tag) {
+ return rewrite(tag)
+ }
+ var decoded string
+ if err := json.Unmarshal([]byte(`"`+tag+`"`), &decoded); err != nil {
+ return ""
+ }
+ var encoded strings.Builder
+ encoder := json.NewEncoder(&encoded)
+ // Keep angle brackets visible to the citation scanner when this output is replayed.
+ encoder.SetEscapeHTML(false)
+ _ = encoder.Encode(rewrite(decoded))
+ return encoded.String()[1 : encoded.Len()-2] // Remove the JSON quotes and encoder newline.
+ }
}
func publicAttr(expression *regexp.Regexp, tag string) string {
@@ -146,6 +166,8 @@ var (
modelWebTagRE = regexp.MustCompile(`(?is)<web(?:\s|$)[^>]*(?:>|$)`)
)
+var jsonCitationTagRE = regexp.MustCompile(`(?i)^<(?:ref|kb|web)\s+[a-z_]+\s*=\s*\\"`)
+
var (
documentAttrRE = regexp.MustCompile(`(?i)\bknowledge_id\s*=\s*"([^"]+)"`)
documentElementRE = regexp.MustCompile(`(?is)<knowledge_id>\s*([^<]+?)\s*</knowledge_id>`)
@@ -189,13 +211,13 @@ func (r *sourceRegistry) ExpandText(text string) string {
if !r.citationsEnabled {
return refCandidateRE.ReplaceAllString(text, "")
}
- return refCandidateRE.ReplaceAllStringFunc(text, func(tag string) string {
+ return refCandidateRE.ReplaceAllStringFunc(text, withJSONCitationEscaping(func(tag string) string {
match := refTagRE.FindStringSubmatch(tag)
if len(match) != 2 {
return ""
}
return r.citationTag(match[1])
- })
+ }))
}
// citationTag renders the public citation for a handle backed by current
@@ -261,9 +283,7 @@ func (d *citationStreamExpander) Feed(chunk string) string {
break
}
tag := data[:end+1]
- if refTagRE.MatchString(tag) {
- out.WriteString(d.registry.ExpandText(tag))
- }
+ out.WriteString(d.registry.ExpandText(tag))
data = data[end+1:]
continue
}
diff --git a/internal/modelcontext/sources_test.go b/internal/modelcontext/sources_test.go
index ce66ac83f..f8f319cd2 100644
--- a/internal/modelcontext/sources_test.go
+++ b/internal/modelcontext/sources_test.go
@@ -1,6 +1,7 @@
package modelcontext
import (
+ "encoding/json"
"fmt"
"strings"
"testing"
@@ -10,6 +11,86 @@ import (
"github.com/stretchr/testify/require"
)
+func TestCitationsInJSONStrings(t *testing.T) {
+ registry := NewRegistry(true)
+ ref := ChunkReference{
+ ChunkID: "chunk-real", KnowledgeBaseID: "kb-real", DocumentTitle: "Guide \"quoted\" & <text> \\ notes\nnext",
+ }
+ registry.RegisterChunk(ref)
+ registry.RegisterWeb("https://example.com/?a=1&b=2", ref.DocumentTitle)
+ input := `{"chunk":"<ref id=\"chunk://1\"/>","web":"<ref id=\"url://1\"/>","text":"keep \\ and \"quotes\""}`
+ output := registry.DecodeOutputText(input)
+ var decoded map[string]string
+ require.NoError(t, json.Unmarshal([]byte(output), &decoded))
+ require.Equal(t, registry.DecodeOutputText(`<ref id="chunk://1"/>`), decoded["chunk"])
+ require.Equal(t, registry.DecodeOutputText(`<ref id="url://1"/>`), decoded["web"])
+ require.Equal(t, `keep \ and "quotes"`, decoded["text"])
+ for split := 0; split <= len(input); split++ {
+ decoder := registry.StreamDecoder()
+ got := decoder.Feed(input[:split]) + decoder.Feed(input[split:]) + decoder.Flush()
+ require.Equal(t, output, got, "split at %d", split)
+ }
+ decoder := registry.StreamDecoder()
+ var streamed strings.Builder
+ for i := range len(input) {
+ streamed.WriteString(decoder.Feed(input[i : i+1]))
+ }
+ streamed.WriteString(decoder.Flush())
+ require.Equal(t, output, streamed.String())
+
+ // Replayed citations remain navigation-only until a current retrieval supplies evidence.
+ history := NewRegistry(true)
+ messages := history.EncodeMessages([]chat.Message{{Role: "assistant", Content: output}})
+ require.JSONEq(t, input, messages[0].Content)
+ require.NoError(t, json.Unmarshal([]byte(history.DecodeOutputText(messages[0].Content)), &decoded))
+ require.Empty(t, decoded["chunk"])
+ require.Empty(t, decoded["web"])
+ history.RegisterChunk(ref)
+ history.RegisterWeb("https://example.com/?a=1&b=2", ref.DocumentTitle)
+ require.JSONEq(t, output, history.DecodeOutputText(messages[0].Content))
+}
+
+func TestJSONCitationsPreserveSourcePolicy(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ citation string
+ disabled bool
+ }{
+ {"unknown", `<ref id=\"chunk://999\"/>`, false},
+ {"history only", `<ref id=\"chunk://2\"/>`, false},
+ {"document", `<ref id=\"doc://1\"/>`, false},
+ {"knowledge base", `<ref id=\"kb://1\"/>`, false},
+ {"disabled chunk", `<ref id=\"chunk://1\"/>`, true},
+ {"disabled web", `<ref id=\"url://1\"/>`, true},
+ {"forged chunk", `<kb chunk_id=\"chunk-real\"/>`, false},
+ {"forged web", `<web url=\"https://example.com\"/>`, false},
+ {"extra attribute", `<ref id=\"chunk://1\" other=\"value\"/>`, false},
+ {"double encoding", `<ref id=\\\"chunk://1\\\"/>`, false},
+ {"nested ref", `<ref id=\"<ref id=\\\"chunk://1\\\"/>\"/>`, false},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ registry := NewRegistry(!tc.disabled)
+ registry.RegisterChunk(ChunkReference{ChunkID: "chunk-real"})
+ registry.RegisterContextChunk(ChunkReference{ChunkID: "history-real"})
+ registry.RegisterDocument("doc-real")
+ registry.RegisterKnowledgeBase("kb-real")
+ registry.RegisterWeb("https://example.com", "Example")
+ input := `{"citation":"` + tc.citation + `"}`
+ require.True(t, json.Valid([]byte(input)))
+ output := registry.DecodeOutputText(input)
+ var decoded map[string]string
+ require.NoError(t, json.Unmarshal([]byte(output), &decoded))
+ require.NotContains(t, decoded["citation"], "<kb")
+ require.NotContains(t, decoded["citation"], "<web")
+ for split := 0; split <= len(input); split++ {
+ decoder := registry.StreamDecoder()
+ got := decoder.Feed(input[:split]) + decoder.Feed(input[split:]) + decoder.Flush()
+ require.Equal(t, output, got, "split at %d", split)
+ }
+ })
+ }
+}
+
func TestRegistryChunkAliasIsStableAndExpandsCanonicalCitation(t *testing.T) {
registry := newSourceRegistry()
first := registry.RegisterChunk(ChunkReference{