Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(rerank): preserve literal fenced code in model passages (#4521)

Wang Xinming1bcc004a5e5e
diff --git a/internal/reranking/passage.go b/internal/reranking/passage.go
index 364d3c13a..fd3216568 100644
--- a/internal/reranking/passage.go
+++ b/internal/reranking/passage.go
@@ -32,8 +32,6 @@ var (
reMarkdownLink = regexp.MustCompile(`\[([^\]]+)\]\([^()\s]*(?:\([^)]*\)[^()\s]*)*\)`)
// reRawURL matches standalone http(s) URLs.
reRawURL = regexp.MustCompile(`https?://[^\s)\]>]+`)
- // reCodeBlock captures the semantic body of fenced code blocks.
- reCodeBlock = regexp.MustCompile("(?s)```[^\\r\\n]*\\r?\\n(.*?)\\r?\\n?```")
// reLatexBlock captures the semantic body of block-level LaTeX ($$...$$).
reLatexBlock = regexp.MustCompile(`(?s)\$\$(.*?)\$\$`)
// reTableSep matches table separator rows like |---|---|.
@@ -65,8 +63,59 @@ var (
// all meaningful semantic content while removing formatting that would
// confuse text-similarity scoring.
func CleanPassage(text string) string {
- // 1. Unwrap code blocks so code-only candidates remain rerankable.
- text = reCodeBlock.ReplaceAllString(text, "$1")
+ var out, prose strings.Builder
+ var fenceChar byte
+ fenceLength := 0
+ flushProse := func() {
+ if prose.Len() == 0 {
+ return
+ }
+ out.WriteString(cleanPassageProse(prose.String()))
+ prose.Reset()
+ }
+
+ // Keep fenced code separate from prose throughout cleaning. Unwrapping
+ // first would let later passes delete XML, URLs and operators in code.
+ for line := range strings.SplitAfterSeq(text, "\n") {
+ marker, length, tail := passageFence(line)
+ if fenceLength == 0 {
+ if length < 3 || (marker == '`' && strings.ContainsRune(tail, '`')) {
+ prose.WriteString(line)
+ continue
+ }
+ flushProse()
+ fenceChar, fenceLength = marker, length
+ } else if marker == fenceChar && length >= fenceLength && strings.Trim(tail, " \t\r\n") == "" {
+ fenceLength = 0
+ } else {
+ out.WriteString(line)
+ }
+ }
+ flushProse()
+ return strings.TrimSpace(out.String())
+}
+
+// passageFence recognizes a top-level fence with up to three leading spaces.
+// Matching type/length and a whitespace-only closing tail follow CommonMark
+// 4.5. This helper does not recognize container, indented or inline code.
+func passageFence(line string) (marker byte, length int, tail string) {
+ trimmed := strings.TrimLeft(line, " ")
+ if len(line)-len(trimmed) > 3 || len(trimmed) == 0 {
+ return 0, 0, ""
+ }
+ marker = trimmed[0]
+ if marker != '`' && marker != '~' {
+ return 0, 0, ""
+ }
+ for length < len(trimmed) && trimmed[length] == marker {
+ length++
+ }
+ return marker, length, trimmed[length:]
+}
+
+// cleanPassageProse leaves boundary whitespace intact so code and surrounding
+// text retain their line boundaries when the cleaned segments are assembled.
+func cleanPassageProse(text string) string {
// 2. Unwrap LaTeX blocks so formula-only candidates remain rerankable.
text = reLatexBlock.ReplaceAllString(text, "$1")
// 3. Remove HTML tags
@@ -111,7 +160,7 @@ func CleanPassage(text string) string {
// 12. Collapse excessive newlines
text = reExcessiveNewlines.ReplaceAllString(text, "\n\n")
- return strings.TrimSpace(text)
+ return text
}
// EnrichedPassage is the cleaned chunk body plus the image captions, OCR text
diff --git a/internal/reranking/passage_code_test.go b/internal/reranking/passage_code_test.go
new file mode 100644
index 000000000..c9720196e
--- /dev/null
+++ b/internal/reranking/passage_code_test.go
@@ -0,0 +1,132 @@
+package reranking
+
+import (
+ "context"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/types"
+)
+
+func TestCleanPassagePreservesFencedCode(t *testing.T) {
+ t.Parallel()
+ tests := []struct {
+ name, input, want string
+ }{
+ {
+ name: "XML-only evidence must not become empty",
+ input: "```xml\n<refund-policy days=\"7\"/>\n```",
+ want: "<refund-policy days=\"7\"/>",
+ },
+ {
+ name: "URLs and formatting-like operators are code",
+ input: "```python\nurl = \"https://api.example.com/v1\"\nvalue = 2 ** 3 ** 2\n```",
+ want: "url = \"https://api.example.com/v1\"\nvalue = 2 ** 3 ** 2",
+ },
+ {
+ name: "literal Markdown is code",
+ input: "```markdown\n# Heading\n- item\n| A | B |\n| --- | --- |\n" +
+ "![alt](image.png)\n[label](https://example.com)\n```",
+ want: "# Heading\n- item\n| A | B |\n| --- | --- |\n" +
+ "![alt](image.png)\n[label](https://example.com)",
+ },
+ {
+ name: "code blank lines and math delimiters survive",
+ input: "```text\nbefore\n\n\n\nafter $$price$$\n```",
+ want: "before\n\n\n\nafter $$price$$",
+ },
+ {
+ name: "longer outer fence keeps shorter inner fence",
+ input: "````markdown\n```xml\n<policy/>\n```\n````",
+ want: "```xml\n<policy/>\n```",
+ },
+ {
+ name: "tilde fences",
+ input: "~~~xml\n<policy/>\n~~~",
+ want: "<policy/>",
+ },
+ {
+ name: "mismatched delimiter is code",
+ input: "```text\n~~~\n<policy/>\n```",
+ want: "~~~\n<policy/>",
+ },
+ {
+ name: "closing fence cannot carry an info string",
+ input: "```text\n```not-a-close\n<policy/>\n```",
+ want: "```not-a-close\n<policy/>",
+ },
+ {
+ name: "longer closing fence",
+ input: "```xml\n<policy/>\n`````",
+ want: "<policy/>",
+ },
+ {
+ name: "unclosed fence protects remaining code",
+ input: "```xml\n<policy/>\nhttps://api.example.com",
+ want: "<policy/>\nhttps://api.example.com",
+ },
+ {
+ name: "CRLF code lines",
+ input: "```xml\r\n<one/>\r\n<two/>\r\n```\r\n",
+ want: "<one/>\r\n<two/>",
+ },
+ {
+ name: "indented fence and CRLF closing whitespace",
+ input: " ~~~xml\r\n<policy/>\r\n ~~~ \t\r\n",
+ want: "<policy/>",
+ },
+ {
+ name: "only prose is cleaned between code blocks",
+ input: "## Example\n```xml\n<one/>\n```\n**Explanation** [docs](https://example.com)\n" +
+ "~~~shell\ncurl https://api.example.com\n~~~\n> End",
+ want: "Example\n<one/>\nExplanation docs\ncurl https://api.example.com\nEnd",
+ },
+ {
+ name: "inline triple backticks do not open a block",
+ input: "Text ```not a block\n**bold**",
+ want: "Text ```not a block\nbold",
+ },
+ {
+ name: "four-space indented fences are not fenced blocks",
+ input: " ```xml\n<policy/>\n ```",
+ want: "```xml\n\n ```",
+ },
+ {
+ name: "backticks in a backtick info string invalidate the opening",
+ input: "```bad`info\n**bold**",
+ want: "```bad`info\nbold",
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := CleanPassage(tt.input); got != tt.want {
+ t.Errorf("CleanPassage() = %q, want %q", got, tt.want)
+ }
+ })
+ }
+}
+
+func TestRerankReceivesFencedCodeWithoutMarkdownCleaning(t *testing.T) {
+ t.Parallel()
+ content := "```xml\n<refund-policy days=\"7\"/>\n```"
+ model := &stubReranker{scores: []float64{0.9}}
+ input := []*types.SearchResult{{
+ ID: "policy", Content: content, KnowledgeTitle: "API policy",
+ ContextHeader: "Refunds", ChunkType: string(types.ChunkTypeText), Score: 0.8,
+ }}
+ result := Rerank(context.Background(), model, "refund policy days", input,
+ Options{Threshold: 0.3})
+
+ want := "API policy\n\nRefunds\n\n<refund-policy days=\"7\"/>"
+ if len(model.documents) != 1 || model.documents[0] != want {
+ t.Fatalf("model received %q, want one passage %q", model.documents, want)
+ }
+ if len(result.Results) != 1 || result.Results[0].ID != "policy" {
+ t.Fatalf("code-only candidate was dropped: %+v", result)
+ }
+ if input[0].Content != content || result.Results[0].Content != content {
+ t.Fatal("passage cleaning changed the stored/returned chunk content")
+ }
+ if got := EnrichedPassage(context.Background(), input[0]); got != "<refund-policy days=\"7\"/>" {
+ t.Fatalf("MMR enrichment lost the code evidence: %q", got)
+ }
+}
diff --git a/website-docs/02-architecture/04-rag-pipeline.md b/website-docs/02-architecture/04-rag-pipeline.md
index 6abafa66c..37fd0785b 100644
--- a/website-docs/02-architecture/04-rag-pipeline.md
+++ b/website-docs/02-architecture/04-rag-pipeline.md
@@ -216,7 +216,7 @@ pipeline = types.NewPipelineBuilder().
**PluginRerank**(`rerank.go`,720 行):
-1. **Passage 清洗** `cleanPassageForRerank`:重排模型做的是语义相似度,Markdown 结构语法是噪声。代码块与 `$$...$$` 公式块**只脱掉围栏、保留内部正文**(早期实现整块删除,纯代码或纯公式的候选会被清成空串而丢分);HTML 标签、图片引用、链接标记(保留文字)、裸 URL、表格分隔行(数据行转逗号拼接)、标题/引用/加粗/列表标记按序剥离,最后压缩多余空行。
+1. **Passage 清洗** `reranking.CleanPassage`:重排模型做的是语义相似度,Markdown 结构语法是噪声。顶层反引号或波浪线代码围栏只移除开闭标记,正文不再经过 HTML、URL 或 Markdown 清洗,避免 XML 标签、接口地址和代码运算符被删除;匹配围栏类型和长度,未闭合围栏保护到分块末尾。这里只识别至多三个前导空格的顶层围栏,不覆盖行内代码、缩进代码或引用/列表容器中的代码块。围栏外的 `$$...$$` 公式块脱掉定界符、保留正文;HTML 标签、图片引用、链接标记(保留文字)、裸 URL、表格分隔行(数据行转逗号拼接)、标题/引用/加粗/列表标记按序剥离,并压缩围栏外的多余空行。整个 passage 仍去除首尾空白;原始分块正文不变。
2. **Passage 增强** `getEnrichedPassage`:拼入 `ImageInfo` 的 Caption/OCR 文本与 `ChunkMetadata` 中的生成问题(GeneratedQuestions);与正文相同的图片文本(OCR / 描述分块自身)不重复拼接。模型配置了单篇或单次请求长度上限时,超长 passage 从尾部截到上限;候选按检索分最多取 200 条送去打分。
3. 调用 `rerankModel.Rerank(ctx, RewriteQuery, passages)`,按 `RerankThreshold` 过滤:
- 全部低于阈值且 top1 ≥ `rerankFallbackMinScore`(默认 0.15;用户显式圈定标签/文档范围时为 0,保留权威范围的最佳候选)→ 保留 top1 兜底;