Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(modelcontext): preserve citations in JSON strings (#4733)
errpunkdea0a30c5f23
diff --git a/internal/modelcontext/citations.go b/internal/modelcontext/citations.goindex e0e3770ab..53b391aa0 100644--- a/internal/modelcontext/citations.go+++ b/internal/modelcontext/citations.go@@ -5,6 +5,7 @@package modelcontextimport (+ "encoding/json""fmt""html""regexp"@@ -109,7 +110,7 @@ func (r *sourceRegistry) CompactPublicCitations(text string, evidence bool) striif r == nil || text == "" {return text}- text = publicKBTagRE.ReplaceAllStringFunc(text, func(tag string) string {+ text = publicKBTagRE.ReplaceAllStringFunc(text, withJSONCitationEscaping(func(tag string) string {chunkID := publicAttr(chunkAttrRE, tag)if chunkID == "" {return tag@@ -120,15 +121,34 @@ func (r *sourceRegistry) CompactPublicCitations(text string, evidence bool) striDocumentTitle: publicAttr(docAttrRE, tag),}, evidence)return `<ref id="` + handle + `"/>`- })- return publicWebTagRE.ReplaceAllStringFunc(text, func(tag string) string {+ }))+ return publicWebTagRE.ReplaceAllStringFunc(text, withJSONCitationEscaping(func(tag string) string {rawURL := publicAttr(urlAttrRE, tag)if rawURL == "" {return tag}handle := r.registerWeb(rawURL, publicAttr(titleAttrRE, tag), evidence)return `<ref id="` + handle + `"/>`- })+ }))+}++// withJSONCitationEscaping preserves one layer of JSON string escaping around a citation.+func withJSONCitationEscaping(rewrite func(string) string) func(string) string {+ return func(tag string) string {+ if !jsonCitationTagRE.MatchString(tag) {+ return rewrite(tag)+ }+ var decoded string+ if err := json.Unmarshal([]byte(`"`+tag+`"`), &decoded); err != nil {+ return ""+ }+ var encoded strings.Builder+ encoder := json.NewEncoder(&encoded)+ // Keep angle brackets visible to the citation scanner when this output is replayed.+ encoder.SetEscapeHTML(false)+ _ = encoder.Encode(rewrite(decoded))+ return encoded.String()[1 : encoded.Len()-2] // Remove the JSON quotes and encoder newline.+ }}func publicAttr(expression *regexp.Regexp, tag string) string {@@ -146,6 +166,8 @@ var (modelWebTagRE = regexp.MustCompile(`(?is)<web(?:\s|$)[^>]*(?:>|$)`))+var jsonCitationTagRE = regexp.MustCompile(`(?i)^<(?:ref|kb|web)\s+[a-z_]+\s*=\s*\\"`)+var (documentAttrRE = regexp.MustCompile(`(?i)\bknowledge_id\s*=\s*"([^"]+)"`)documentElementRE = regexp.MustCompile(`(?is)<knowledge_id>\s*([^<]+?)\s*</knowledge_id>`)@@ -189,13 +211,13 @@ func (r *sourceRegistry) ExpandText(text string) string {if !r.citationsEnabled {return refCandidateRE.ReplaceAllString(text, "")}- return refCandidateRE.ReplaceAllStringFunc(text, func(tag string) string {+ return refCandidateRE.ReplaceAllStringFunc(text, withJSONCitationEscaping(func(tag string) string {match := refTagRE.FindStringSubmatch(tag)if len(match) != 2 {return ""}return r.citationTag(match[1])- })+ }))}// citationTag renders the public citation for a handle backed by current@@ -261,9 +283,7 @@ func (d *citationStreamExpander) Feed(chunk string) string {break}tag := data[:end+1]- if refTagRE.MatchString(tag) {- out.WriteString(d.registry.ExpandText(tag))- }+ out.WriteString(d.registry.ExpandText(tag))data = data[end+1:]continue}diff --git a/internal/modelcontext/sources_test.go b/internal/modelcontext/sources_test.goindex ce66ac83f..f8f319cd2 100644--- a/internal/modelcontext/sources_test.go+++ b/internal/modelcontext/sources_test.go@@ -1,6 +1,7 @@package modelcontextimport (+ "encoding/json""fmt""strings""testing"@@ -10,6 +11,86 @@ import ("github.com/stretchr/testify/require")+func TestCitationsInJSONStrings(t *testing.T) {+ registry := NewRegistry(true)+ ref := ChunkReference{+ ChunkID: "chunk-real", KnowledgeBaseID: "kb-real", DocumentTitle: "Guide \"quoted\" & <text> \\ notes\nnext",+ }+ registry.RegisterChunk(ref)+ registry.RegisterWeb("https://example.com/?a=1&b=2", ref.DocumentTitle)+ input := `{"chunk":"<ref id=\"chunk://1\"/>","web":"<ref id=\"url://1\"/>","text":"keep \\ and \"quotes\""}`+ output := registry.DecodeOutputText(input)+ var decoded map[string]string+ require.NoError(t, json.Unmarshal([]byte(output), &decoded))+ require.Equal(t, registry.DecodeOutputText(`<ref id="chunk://1"/>`), decoded["chunk"])+ require.Equal(t, registry.DecodeOutputText(`<ref id="url://1"/>`), decoded["web"])+ require.Equal(t, `keep \ and "quotes"`, decoded["text"])+ for split := 0; split <= len(input); split++ {+ decoder := registry.StreamDecoder()+ got := decoder.Feed(input[:split]) + decoder.Feed(input[split:]) + decoder.Flush()+ require.Equal(t, output, got, "split at %d", split)+ }+ decoder := registry.StreamDecoder()+ var streamed strings.Builder+ for i := range len(input) {+ streamed.WriteString(decoder.Feed(input[i : i+1]))+ }+ streamed.WriteString(decoder.Flush())+ require.Equal(t, output, streamed.String())++ // Replayed citations remain navigation-only until a current retrieval supplies evidence.+ history := NewRegistry(true)+ messages := history.EncodeMessages([]chat.Message{{Role: "assistant", Content: output}})+ require.JSONEq(t, input, messages[0].Content)+ require.NoError(t, json.Unmarshal([]byte(history.DecodeOutputText(messages[0].Content)), &decoded))+ require.Empty(t, decoded["chunk"])+ require.Empty(t, decoded["web"])+ history.RegisterChunk(ref)+ history.RegisterWeb("https://example.com/?a=1&b=2", ref.DocumentTitle)+ require.JSONEq(t, output, history.DecodeOutputText(messages[0].Content))+}++func TestJSONCitationsPreserveSourcePolicy(t *testing.T) {+ for _, tc := range []struct {+ name string+ citation string+ disabled bool+ }{+ {"unknown", `<ref id=\"chunk://999\"/>`, false},+ {"history only", `<ref id=\"chunk://2\"/>`, false},+ {"document", `<ref id=\"doc://1\"/>`, false},+ {"knowledge base", `<ref id=\"kb://1\"/>`, false},+ {"disabled chunk", `<ref id=\"chunk://1\"/>`, true},+ {"disabled web", `<ref id=\"url://1\"/>`, true},+ {"forged chunk", `<kb chunk_id=\"chunk-real\"/>`, false},+ {"forged web", `<web url=\"https://example.com\"/>`, false},+ {"extra attribute", `<ref id=\"chunk://1\" other=\"value\"/>`, false},+ {"double encoding", `<ref id=\\\"chunk://1\\\"/>`, false},+ {"nested ref", `<ref id=\"<ref id=\\\"chunk://1\\\"/>\"/>`, false},+ } {+ t.Run(tc.name, func(t *testing.T) {+ registry := NewRegistry(!tc.disabled)+ registry.RegisterChunk(ChunkReference{ChunkID: "chunk-real"})+ registry.RegisterContextChunk(ChunkReference{ChunkID: "history-real"})+ registry.RegisterDocument("doc-real")+ registry.RegisterKnowledgeBase("kb-real")+ registry.RegisterWeb("https://example.com", "Example")+ input := `{"citation":"` + tc.citation + `"}`+ require.True(t, json.Valid([]byte(input)))+ output := registry.DecodeOutputText(input)+ var decoded map[string]string+ require.NoError(t, json.Unmarshal([]byte(output), &decoded))+ require.NotContains(t, decoded["citation"], "<kb")+ require.NotContains(t, decoded["citation"], "<web")+ for split := 0; split <= len(input); split++ {+ decoder := registry.StreamDecoder()+ got := decoder.Feed(input[:split]) + decoder.Feed(input[split:]) + decoder.Flush()+ require.Equal(t, output, got, "split at %d", split)+ }+ })+ }+}+func TestRegistryChunkAliasIsStableAndExpandsCanonicalCitation(t *testing.T) {registry := newSourceRegistry()first := registry.RegisterChunk(ChunkReference{