Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(rerank): preserve literal fenced code in model passages (#4521)
Wang Xinming1bcc004a5e5e
diff --git a/internal/reranking/passage.go b/internal/reranking/passage.goindex 364d3c13a..fd3216568 100644--- a/internal/reranking/passage.go+++ b/internal/reranking/passage.go@@ -32,8 +32,6 @@ var (reMarkdownLink = regexp.MustCompile(`\[([^\]]+)\]\([^()\s]*(?:\([^)]*\)[^()\s]*)*\)`)// reRawURL matches standalone http(s) URLs.reRawURL = regexp.MustCompile(`https?://[^\s)\]>]+`)- // reCodeBlock captures the semantic body of fenced code blocks.- reCodeBlock = regexp.MustCompile("(?s)```[^\\r\\n]*\\r?\\n(.*?)\\r?\\n?```")// reLatexBlock captures the semantic body of block-level LaTeX ($$...$$).reLatexBlock = regexp.MustCompile(`(?s)\$\$(.*?)\$\$`)// reTableSep matches table separator rows like |---|---|.@@ -65,8 +63,59 @@ var (// all meaningful semantic content while removing formatting that would// confuse text-similarity scoring.func CleanPassage(text string) string {- // 1. Unwrap code blocks so code-only candidates remain rerankable.- text = reCodeBlock.ReplaceAllString(text, "$1")+ var out, prose strings.Builder+ var fenceChar byte+ fenceLength := 0+ flushProse := func() {+ if prose.Len() == 0 {+ return+ }+ out.WriteString(cleanPassageProse(prose.String()))+ prose.Reset()+ }++ // Keep fenced code separate from prose throughout cleaning. Unwrapping+ // first would let later passes delete XML, URLs and operators in code.+ for line := range strings.SplitAfterSeq(text, "\n") {+ marker, length, tail := passageFence(line)+ if fenceLength == 0 {+ if length < 3 || (marker == '`' && strings.ContainsRune(tail, '`')) {+ prose.WriteString(line)+ continue+ }+ flushProse()+ fenceChar, fenceLength = marker, length+ } else if marker == fenceChar && length >= fenceLength && strings.Trim(tail, " \t\r\n") == "" {+ fenceLength = 0+ } else {+ out.WriteString(line)+ }+ }+ flushProse()+ return strings.TrimSpace(out.String())+}++// passageFence recognizes a top-level fence with up to three leading spaces.+// Matching type/length and a whitespace-only closing tail follow CommonMark+// 4.5. This helper does not recognize container, indented or inline code.+func passageFence(line string) (marker byte, length int, tail string) {+ trimmed := strings.TrimLeft(line, " ")+ if len(line)-len(trimmed) > 3 || len(trimmed) == 0 {+ return 0, 0, ""+ }+ marker = trimmed[0]+ if marker != '`' && marker != '~' {+ return 0, 0, ""+ }+ for length < len(trimmed) && trimmed[length] == marker {+ length+++ }+ return marker, length, trimmed[length:]+}++// cleanPassageProse leaves boundary whitespace intact so code and surrounding+// text retain their line boundaries when the cleaned segments are assembled.+func cleanPassageProse(text string) string {// 2. Unwrap LaTeX blocks so formula-only candidates remain rerankable.text = reLatexBlock.ReplaceAllString(text, "$1")// 3. Remove HTML tags@@ -111,7 +160,7 @@ func CleanPassage(text string) string {// 12. Collapse excessive newlinestext = reExcessiveNewlines.ReplaceAllString(text, "\n\n")- return strings.TrimSpace(text)+ return text}// EnrichedPassage is the cleaned chunk body plus the image captions, OCR textdiff --git a/internal/reranking/passage_code_test.go b/internal/reranking/passage_code_test.gonew file mode 100644index 000000000..c9720196e--- /dev/null+++ b/internal/reranking/passage_code_test.go@@ -0,0 +1,132 @@+package reranking++import (+ "context"+ "testing"++ "github.com/Tencent/WeKnora/internal/types"+)++func TestCleanPassagePreservesFencedCode(t *testing.T) {+ t.Parallel()+ tests := []struct {+ name, input, want string+ }{+ {+ name: "XML-only evidence must not become empty",+ input: "```xml\n<refund-policy days=\"7\"/>\n```",+ want: "<refund-policy days=\"7\"/>",+ },+ {+ name: "URLs and formatting-like operators are code",+ input: "```python\nurl = \"https://api.example.com/v1\"\nvalue = 2 ** 3 ** 2\n```",+ want: "url = \"https://api.example.com/v1\"\nvalue = 2 ** 3 ** 2",+ },+ {+ name: "literal Markdown is code",+ input: "```markdown\n# Heading\n- item\n| A | B |\n| --- | --- |\n" ++ "\n[label](https://example.com)\n```",+ want: "# Heading\n- item\n| A | B |\n| --- | --- |\n" ++ "\n[label](https://example.com)",+ },+ {+ name: "code blank lines and math delimiters survive",+ input: "```text\nbefore\n\n\n\nafter $$price$$\n```",+ want: "before\n\n\n\nafter $$price$$",+ },+ {+ name: "longer outer fence keeps shorter inner fence",+ input: "````markdown\n```xml\n<policy/>\n```\n````",+ want: "```xml\n<policy/>\n```",+ },+ {+ name: "tilde fences",+ input: "~~~xml\n<policy/>\n~~~",+ want: "<policy/>",+ },+ {+ name: "mismatched delimiter is code",+ input: "```text\n~~~\n<policy/>\n```",+ want: "~~~\n<policy/>",+ },+ {+ name: "closing fence cannot carry an info string",+ input: "```text\n```not-a-close\n<policy/>\n```",+ want: "```not-a-close\n<policy/>",+ },+ {+ name: "longer closing fence",+ input: "```xml\n<policy/>\n`````",+ want: "<policy/>",+ },+ {+ name: "unclosed fence protects remaining code",+ input: "```xml\n<policy/>\nhttps://api.example.com",+ want: "<policy/>\nhttps://api.example.com",+ },+ {+ name: "CRLF code lines",+ input: "```xml\r\n<one/>\r\n<two/>\r\n```\r\n",+ want: "<one/>\r\n<two/>",+ },+ {+ name: "indented fence and CRLF closing whitespace",+ input: " ~~~xml\r\n<policy/>\r\n ~~~ \t\r\n",+ want: "<policy/>",+ },+ {+ name: "only prose is cleaned between code blocks",+ input: "## Example\n```xml\n<one/>\n```\n**Explanation** [docs](https://example.com)\n" ++ "~~~shell\ncurl https://api.example.com\n~~~\n> End",+ want: "Example\n<one/>\nExplanation docs\ncurl https://api.example.com\nEnd",+ },+ {+ name: "inline triple backticks do not open a block",+ input: "Text ```not a block\n**bold**",+ want: "Text ```not a block\nbold",+ },+ {+ name: "four-space indented fences are not fenced blocks",+ input: " ```xml\n<policy/>\n ```",+ want: "```xml\n\n ```",+ },+ {+ name: "backticks in a backtick info string invalidate the opening",+ input: "```bad`info\n**bold**",+ want: "```bad`info\nbold",+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ if got := CleanPassage(tt.input); got != tt.want {+ t.Errorf("CleanPassage() = %q, want %q", got, tt.want)+ }+ })+ }+}++func TestRerankReceivesFencedCodeWithoutMarkdownCleaning(t *testing.T) {+ t.Parallel()+ content := "```xml\n<refund-policy days=\"7\"/>\n```"+ model := &stubReranker{scores: []float64{0.9}}+ input := []*types.SearchResult{{+ ID: "policy", Content: content, KnowledgeTitle: "API policy",+ ContextHeader: "Refunds", ChunkType: string(types.ChunkTypeText), Score: 0.8,+ }}+ result := Rerank(context.Background(), model, "refund policy days", input,+ Options{Threshold: 0.3})++ want := "API policy\n\nRefunds\n\n<refund-policy days=\"7\"/>"+ if len(model.documents) != 1 || model.documents[0] != want {+ t.Fatalf("model received %q, want one passage %q", model.documents, want)+ }+ if len(result.Results) != 1 || result.Results[0].ID != "policy" {+ t.Fatalf("code-only candidate was dropped: %+v", result)+ }+ if input[0].Content != content || result.Results[0].Content != content {+ t.Fatal("passage cleaning changed the stored/returned chunk content")+ }+ if got := EnrichedPassage(context.Background(), input[0]); got != "<refund-policy days=\"7\"/>" {+ t.Fatalf("MMR enrichment lost the code evidence: %q", got)+ }+}diff --git a/website-docs/02-architecture/04-rag-pipeline.md b/website-docs/02-architecture/04-rag-pipeline.mdindex 6abafa66c..37fd0785b 100644--- a/website-docs/02-architecture/04-rag-pipeline.md+++ b/website-docs/02-architecture/04-rag-pipeline.md@@ -216,7 +216,7 @@ pipeline = types.NewPipelineBuilder().**PluginRerank**(`rerank.go`,720 行):-1. **Passage 清洗** `cleanPassageForRerank`:重排模型做的是语义相似度,Markdown 结构语法是噪声。代码块与 `$$...$$` 公式块**只脱掉围栏、保留内部正文**(早期实现整块删除,纯代码或纯公式的候选会被清成空串而丢分);HTML 标签、图片引用、链接标记(保留文字)、裸 URL、表格分隔行(数据行转逗号拼接)、标题/引用/加粗/列表标记按序剥离,最后压缩多余空行。+1. **Passage 清洗** `reranking.CleanPassage`:重排模型做的是语义相似度,Markdown 结构语法是噪声。顶层反引号或波浪线代码围栏只移除开闭标记,正文不再经过 HTML、URL 或 Markdown 清洗,避免 XML 标签、接口地址和代码运算符被删除;匹配围栏类型和长度,未闭合围栏保护到分块末尾。这里只识别至多三个前导空格的顶层围栏,不覆盖行内代码、缩进代码或引用/列表容器中的代码块。围栏外的 `$$...$$` 公式块脱掉定界符、保留正文;HTML 标签、图片引用、链接标记(保留文字)、裸 URL、表格分隔行(数据行转逗号拼接)、标题/引用/加粗/列表标记按序剥离,并压缩围栏外的多余空行。整个 passage 仍去除首尾空白;原始分块正文不变。2. **Passage 增强** `getEnrichedPassage`:拼入 `ImageInfo` 的 Caption/OCR 文本与 `ChunkMetadata` 中的生成问题(GeneratedQuestions);与正文相同的图片文本(OCR / 描述分块自身)不重复拼接。模型配置了单篇或单次请求长度上限时,超长 passage 从尾部截到上限;候选按检索分最多取 200 条送去打分。3. 调用 `rerankModel.Rerank(ctx, RewriteQuery, passages)`,按 `RerankThreshold` 过滤:- 全部低于阈值且 top1 ≥ `rerankFallbackMinScore`(默认 0.15;用户显式圈定标签/文档范围时为 0,保留权威范围的最佳候选)→ 保留 top1 兜底;