RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Port PR17015 to GO (#19969)
qinling02104c618856ba93
diff --git a/internal/common/string_utils.go b/internal/common/string_utils.gonew file mode 100644index 0000000000..01fa862647--- /dev/null+++ b/internal/common/string_utils.go@@ -0,0 +1,50 @@+package common++import (+ "regexp"+ "strings"+)++// Redundant-space cleanup regexes. Mirrors+// common.string_utils.remove_redundant_spaces (string_utils.py:20-46).+// Pass 1: drop spaces after a "left boundary" character (parens, <, >).+// Pass 2: drop spaces before a "right boundary" character (parens, !).+//+// The word-character class is Unicode-aware (`\p{L}`, `\p{N}`, `_`), matching+// Python's `\w`. An ASCII-only `a-z0-9` puts every non-Latin letter into the+// negated "punctuation" set, which deletes the space between two words:+// "привет мир" -> "приветмир". Letter case needs no flag because `\p{L}`+// covers both cases — that is what Python's `re.IGNORECASE` does for `a-z`.+//+// The non-space class is a literal space, like Python's `[^ ]`: a tab or+// newline is a boundary character there, not whitespace to skip.+var (+ redundantSpacePass1Re = regexp.MustCompile(`([^\p{L}\p{N}_.,)>]) +([^ ])`)+ redundantSpacePass2Re = regexp.MustCompile(`([^ ]) +([^\p{L}\p{N}_.,(<])`)+)++// RemoveRedundantSpaces removes redundant spaces around punctuation marks+// while preserving the spaces between words.+func RemoveRedundantSpaces(s string) string {+ s = redundantSpacePass1Re.ReplaceAllString(s, "$1$2")+ s = redundantSpacePass2Re.ReplaceAllString(s, "$1$2")+ return s+}++// Markdown-fence recognizers. Mirrors+// common.string_utils.clean_markdown_block (string_utils.py:49-...).+// Matches Python without re.MULTILINE so ^/$ anchor only the whole text.+var (+ markdownFenceOpenRe = regexp.MustCompile(`^\s*` + "```" + `markdown\s*\n?`)+ markdownFenceCloseRe = regexp.MustCompile(`\n?\s*` + "```" + `\s*$`)+)++// CleanMarkdownBlock removes a closing fence only after recognizing an opening+// markdown wrapper; anything else is returned trimmed and otherwise untouched.+func CleanMarkdownBlock(s string) string {+ if markdownFenceOpenRe.MatchString(s) {+ s = markdownFenceOpenRe.ReplaceAllString(s, "")+ s = markdownFenceCloseRe.ReplaceAllString(s, "")+ }+ return strings.TrimSpace(s)+}diff --git a/internal/common/string_utils_test.go b/internal/common/string_utils_test.gonew file mode 100644index 0000000000..86fef7b395--- /dev/null+++ b/internal/common/string_utils_test.go@@ -0,0 +1,148 @@+package common++import "testing"++// TestRemoveRedundantSpaces mirrors common.string_utils.remove_redundant_spaces.+func TestRemoveRedundantSpaces(t *testing.T) {+ cases := []struct {+ name string+ in string+ want string+ }{+ // Both passes run sequentially — pass 1 strips space after `(`,+ // pass 2 strips space before `)`, so both go.+ {"pass1+pass2 on ( world )", "hello ( world )", "hello (world)"},+ // Pass 2 strips space before `!`.+ {"pass2: space before !", "world !", "world!"},+ // Comma is not a boundary in pass 2 (it's in the negated set+ // along with `<` and `(`), so no change.+ {"comma not a boundary", "a , b", "a , b"},+ {"no match", "foo bar", "foo bar"},+ {"empty", "", ""},+ {"digit not a boundary", "abc 123", "abc 123"},+ {"left paren kept (no following space)", "(abc)", "(abc)"},+ // Uppercase letters are word characters too: the Python port compiles+ // with re.IGNORECASE, so "Hello World" must not become "HelloWorld".+ {"uppercase is a word char", "Hello World", "Hello World"},+ {"uppercase initials", "A B", "A B"},+ // Regression: non-Latin letters must not fall into the negated+ // "punctuation" set, otherwise the space between two words is deleted+ // ("привет мир" -> "приветмир").+ {"russian", "привет мир", "привет мир"},+ {"greek", "Γειά σου κόσμε", "Γειά σου κόσμε"},+ {"korean", "안녕 세계", "안녕 세계"},+ {"arabic", "مرحبا بالعالم", "مرحبا بالعالم"},+ {"chinese", "中文 测试 句子", "中文 测试 句子"},+ // Punctuation cleanup still applies to non-Latin text.+ {"accented, space before !", "sécurité des données !", "sécurité des données!"},+ {"space before punctuation is still dropped", "привет !", "привет!"},+ {"full-width parens", "( 中文 )", "(中文)"},+ // The non-space class is a literal space, like Python's `[^ ]`: a+ // newline or tab is a boundary character, not whitespace to skip.+ {"newline is a boundary", "hello \nworld", "hello\nworld"},+ {"tab is a boundary", "foo\t bar", "foo\tbar"},+ {"CRLF run collapses", "x\r\n y", "x\r\ny"},+ // `<` is a left boundary: the space after it goes, the one before stays.+ {"space after < is dropped, before kept", "a < b", "a <b"},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ if got := RemoveRedundantSpaces(tc.in); got != tc.want {+ t.Errorf("RemoveRedundantSpaces(%q) = %q, want %q", tc.in, got, tc.want)+ }+ })+ }+}++// TestCleanMarkdownBlock mirrors common.string_utils.clean_markdown_block.+func TestCleanMarkdownBlock(t *testing.T) {+ cases := []struct {+ name string+ input string+ want string+ }{+ {+ name: "standard markdown block",+ input: "```markdown\nHello world\n```",+ want: "Hello world",+ },+ {+ name: "windows CRLF endings",+ input: "```markdown\r\nHello world\r\n```",+ want: "Hello world",+ },+ {+ name: "nested code blocks preserved",+ input: "```markdown\nText with ```nested``` blocks\n```",+ want: "Text with ```nested``` blocks",+ },+ {+ name: "mixed whitespace tabs",+ input: "\t```markdown\t\n\tContent with tabs\n\t```\t",+ want: "Content with tabs",+ },+ {+ name: "empty string",+ input: "",+ want: "",+ },+ {+ name: "only markdown tags",+ input: "```markdown```",+ want: "",+ },+ {+ name: "unwrapped text unchanged",+ input: "Just plain text without markdown block",+ want: "Just plain text without markdown block",+ },+ {+ name: "unwrapped python block",+ input: "```python\nprint('hello')\n```",+ want: "```python\nprint('hello')\n```",+ },+ {+ name: "unwrapped JSON block",+ input: "```json\n{\"value\": 1}\n```",+ want: "```json\n{\"value\": 1}\n```",+ },+ {+ name: "unlabeled code block",+ input: "```\ncode without a language\n```",+ want: "```\ncode without a language\n```",+ },+ {+ name: "prose ending with code block",+ input: "Example:\n```python\nprint('hello')\n```",+ want: "Example:\n```python\nprint('hello')\n```",+ },+ {+ name: "multiple unwrapped code blocks",+ input: "```python\nfirst()\n```\n\n```python\nsecond()\n```",+ want: "```python\nfirst()\n```\n\n```python\nsecond()\n```",+ },+ {+ name: "unwrapped CRLF block",+ input: " ```python\r\nprint('hello')\r\n``` ",+ want: "```python\r\nprint('hello')\r\n```",+ },+ {+ name: "closing fence without markdown opener",+ input: "Unopened block\n```",+ want: "Unopened block\n```",+ },+ {+ name: "markdown wrapper around code example",+ input: "```markdown\nExample:\n```python\nprint('hello')\n```\n```",+ want: "Example:\n```python\nprint('hello')\n```",+ },+ }++ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ if got := CleanMarkdownBlock(tc.input); got != tc.want {+ t.Errorf("CleanMarkdownBlock(%q) = %q, want %q", tc.input, got, tc.want)+ }+ })+ }+}diff --git a/internal/ingestion/component/vision_enhancement.go b/internal/ingestion/component/vision_enhancement.goindex 7064e1c3d6..f237997713 100644--- a/internal/ingestion/component/vision_enhancement.go+++ b/internal/ingestion/component/vision_enhancement.go@@ -29,11 +29,11 @@ import ("log/slog""os""path/filepath"- "regexp""strings""sync""time"+ "ragflow/internal/common"pdflayout "ragflow/internal/deepdoc/parser/pdf/layout""ragflow/internal/entity"modelModule "ragflow/internal/entity/models"@@ -90,23 +90,6 @@ func (s *promptDirState) resolve(requestedRoot string) (string, error) {return s.root, nil}-// cleanMarkdownBlock mirrors Python common/string_utils.py:clean_markdown_block-// and removes a closing fence only after recognizing an opening markdown wrapper.-//-// Matches Python without re.MULTILINE so ^/$ anchor only the whole text.-var (- reMarkdownOpen = regexp.MustCompile(`^\s*` + "```" + `markdown\s*\n?`)- reMarkdownClose = regexp.MustCompile(`\n?\s*` + "```" + `\s*$`)-)--func cleanMarkdownBlock(s string) string {- if reMarkdownOpen.MatchString(s) {- s = reMarkdownOpen.ReplaceAllString(s, "")- s = reMarkdownClose.ReplaceAllString(s, "")- }- return strings.TrimSpace(s)-}-// isUsableVisionImage reports whether raw carries a valid image data URI or a// base64 payload that can be sent to a vision model.func isUsableVisionImage(raw string) bool {@@ -388,7 +371,7 @@ func extractVisionAnswer(resp *modelModule.ChatResponse) string {if resp == nil || resp.Answer == nil {return ""}- return cleanMarkdownBlock(*resp.Answer)+ return common.CleanMarkdownBlock(*resp.Answer)}func defaultVisionChatInvoker(diff --git a/internal/ingestion/component/vision_enhancement_test.go b/internal/ingestion/component/vision_enhancement_test.goindex 0b36fdb44a..cb68846f30 100644--- a/internal/ingestion/component/vision_enhancement_test.go+++ b/internal/ingestion/component/vision_enhancement_test.go@@ -476,7 +476,8 @@ func TestVisionEnhancement_MoreThanConcurrencyItems(t *testing.T) {}// TestVisionEnhancement_PlainTextResponseNotTruncated verifies that a plain-text-// (no fences) VLM response is returned verbatim — cleanMarkdownBlock must not truncate it.+// (no fences) VLM response is returned verbatim — common.CleanMarkdownBlock+// must not truncate it.func TestVisionEnhancement_PlainTextResponseNotTruncated(t *testing.T) {plain := "A pipeline diagram showing three stages."swapVisionGlobals(t, fakeResolver,@@ -709,7 +710,10 @@ func TestExtractVisionAnswer_CleansMarkdownBlock(t *testing.T) {}}-func TestCleanMarkdownBlock_EdgeCases(t *testing.T) {+// TestExtractVisionAnswer_EdgeCases drives the shared+// common.CleanMarkdownBlock through the vision-answer path; the cleanup+// function itself is pinned in common.TestCleanMarkdownBlock.+func TestExtractVisionAnswer_EdgeCases(t *testing.T) {cases := []struct {name stringinput string@@ -794,10 +798,6 @@ func TestCleanMarkdownBlock_EdgeCases(t *testing.T) {for _, tc := range cases {t.Run(tc.name, func(t *testing.T) {- got := cleanMarkdownBlock(tc.input)- if got != tc.want {- t.Errorf("cleanMarkdownBlock(%q) = %q, want %q", tc.input, got, tc.want)- }resp := &modelModule.ChatResponse{Answer: &tc.input}if got := extractVisionAnswer(resp); got != tc.want {t.Errorf("extractVisionAnswer(%q) = %q, want %q", tc.input, got, tc.want)diff --git a/internal/service/chat_pipeline.go b/internal/service/chat_pipeline.goindex f4a8c6f686..6ecc10ac1e 100644--- a/internal/service/chat_pipeline.go+++ b/internal/service/chat_pipeline.go@@ -4363,23 +4363,6 @@ func normalizeSQL(s string) string {// Python parity helpers (dialog_service.py:56-59, 1238-1309, 1321-1365)// ------------------------------------------------------------------------// Redundant-space cleanup regexes. Mirrors-// common.string_utils.remove_redundant_spaces (string_utils.py:20-46).-// Pass 1: drop spaces after a "left boundary" character (parens, <, >).-// Pass 2: drop spaces before a "right boundary" character (parens, !).-var (- redundantSpacePass1Re = regexp.MustCompile(`([^a-z0-9.,)>\x{ff08}]) +([^ ])`) // left boundary + space + non-space- redundantSpacePass2Re = regexp.MustCompile(`([^ ]) +([^a-z0-9.,(<])`) // non-space + space + right boundary-)--// removeRedundantSpaces ports common.string_utils.remove_redundant_spaces.-// Two-pass regex cleanup; both passes use case-insensitive matching.-func removeRedundantSpaces(s string) string {- s = redundantSpacePass1Re.ReplaceAllString(s, "$1$2")- s = redundantSpacePass2Re.ReplaceAllString(s, "$1$2")- return s-}-// ISO timestamp stripping regex. Mirrors the cleanup at// dialog_service.py:1309. Matches `T13:24:55|` or `T13:24:55.123Z|`.var isoTimestampCellRe = regexp.MustCompile(`T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+Z)?\|`)@@ -4468,7 +4451,7 @@ func chunkKBIDForDoc(rowDict map[string]interface{}, kbIDs []string, docID interfunc cleanCellValue(v interface{}) string {s := fmt.Sprintf("%v", v)s = strings.ReplaceAll(s, "None", " ")- return removeRedundantSpaces(s)+ return common.RemoveRedundantSpaces(s)}// extractSourceColumnIndexes returns, for a set of SQL result rows,diff --git a/internal/service/chat_pipeline_test.go b/internal/service/chat_pipeline_test.goindex c64e5ae6b4..8e44a8f9bd 100644--- a/internal/service/chat_pipeline_test.go+++ b/internal/service/chat_pipeline_test.go@@ -924,35 +924,6 @@ func TestShouldUseWebSearch_RequiresTavilyAndTruthyInternet(t *testing.T) {// --- P5 SQL retrieval parity helpers (Python use_sql alignment) ----// TestRemoveRedundantSpaces mirrors common.string_utils.remove_redundant_spaces.-func TestRemoveRedundantSpaces(t *testing.T) {- cases := []struct {- name string- in string- want string- }{- // Both passes run sequentially — pass 1 strips space after `(`,- // pass 2 strips space before `)`, so both go.- {"pass1+pass2 on ( world )", "hello ( world )", "hello (world)"},- // Pass 2 strips space before `!`.- {"pass2: space before !", "world !", "world!"},- // Comma is not a boundary in pass 2 (it's in the negated set- // along with `<` and `(`), so no change.- {"comma not a boundary", "a , b", "a , b"},- {"no match", "foo bar", "foo bar"},- {"empty", "", ""},- {"digit not a boundary", "abc 123", "abc 123"},- {"left paren kept (no following space)", "(abc)", "(abc)"},- }- for _, tc := range cases {- t.Run(tc.name, func(t *testing.T) {- if got := removeRedundantSpaces(tc.in); got != tc.want {- t.Errorf("removeRedundantSpaces(%q) = %q, want %q", tc.in, got, tc.want)- }- })- }-}-// TestStripISOTimestamps verifies the dialog_service.py:1309 cleanup.// The pattern `T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+Z)?\|` strips the// timestamp + trailing pipe; the leading pipe/space is preserved (thediff --git a/internal/service/nlp/reranker.go b/internal/service/nlp/reranker.goindex 4678ed1c35..3bf8a54f99 100644--- a/internal/service/nlp/reranker.go+++ b/internal/service/nlp/reranker.go@@ -18,7 +18,6 @@ import ("context""encoding/json""math"- "regexp""sort""strconv""strings"@@ -152,13 +151,12 @@ func RerankByModel(// tokenized content_ltks. Neural rerankers score stemmed / accent-split// tokens far lower, which collapses relevance scores and forces an// artificially low similarity_threshold. The natural text is passed- // as-is: RemoveRedundantSpaces is ASCII-oriented and mangles- // multilingual text ("sécurité des données" -> "sécuritédes données"),- // so it is only applied to the tokenized fallback used when- // content_with_weight is absent. Mirrors rag/nlp/search.py.+ // as-is; common.RemoveRedundantSpaces is only applied to the tokenized+ // fallback used when content_with_weight is absent, where it drops the+ // space before punctuation tokens. Mirrors rag/nlp/search.py.docText := extractNaturalText(chunk)if docText == "" {- docText = RemoveRedundantSpaces(strings.Join(tks, " "))+ docText = common.RemoveRedundantSpaces(strings.Join(tks, " "))}docs = append(docs, docText)}@@ -575,7 +573,7 @@ func extractTitleTokens(fields map[string]interface{}) []string {if !ok {return []string{}}- // NOTE: Do NOT call RemoveRedundantSpaces here - it removes spaces between Chinese chars+ // title_tks is a space-separated token list: split it, do not clean it as prose.var result []stringfor t := range strings.FieldsSeq(v) {if t != "" {@@ -652,21 +650,6 @@ func cosineSimilarity(a, b []float64) float64 {return dot / (common.PySqrt(normA) * common.PySqrt(normB))}-// RemoveRedundantSpaces removes redundant spaces from text-// First pass: remove spaces after left-boundary characters-// Second pass: remove spaces before right-boundary characters-func RemoveRedundantSpaces(s string) string {- // First pass: remove spaces after left-boundary characters (opening brackets, etc.)- // e.g., "( text" -> "(text", "【 text" -> "【text"- s = regexp.MustCompile(`([^\sa-z0-9.,\)>]) +([^\s])`).ReplaceAllString(s, "$1$2")-- // Second pass: remove spaces before right-boundary characters (closing brackets, punctuation)- // e.g., "text !" -> "text!"- s = regexp.MustCompile(`([^\s]) +([^\sa-z0-9.,\(])`).ReplaceAllString(s, "$1$2")-- return s-}-// parseFloat parses a string to float64func parseFloat(s string) (float64, error) {return strconv.ParseFloat(strings.TrimSpace(s), 64)diff --git a/internal/service/nlp/reranker_question_tks_test.go b/internal/service/nlp/reranker_question_tks_test.goindex a6a2204396..3fde4ebcf9 100644--- a/internal/service/nlp/reranker_question_tks_test.go+++ b/internal/service/nlp/reranker_question_tks_test.go@@ -130,10 +130,11 @@ func TestRerankByModelPrefersNaturalText(t *testing.T) {}}-// TestRerankByModelNaturalTextKeepsAccentedSpacing guards against-// RemoveRedundantSpaces being applied to natural text: it treats every-// non-ASCII letter as punctuation and would collapse "sécurité des" into-// "sécuritédes".+// TestRerankByModelNaturalTextKeepsAccentedSpacing guards against the+// redundant-space cleanup being applied to natural text: an ASCII-only word+// class treats every non-Latin letter as punctuation and collapses+// "sécurité des" into "sécuritédes". The cleanup itself is pinned in+// common.TestRemoveRedundantSpaces.func TestRerankByModelNaturalTextKeepsAccentedSpacing(t *testing.T) {chunk := fullChunk()chunk["content_with_weight"] = "sécurité des données"@@ -143,9 +144,6 @@ func TestRerankByModelNaturalTextKeepsAccentedSpacing(t *testing.T) {if doc != "sécurité des données" {t.Errorf("natural text was altered: got %q", doc)}- if got := RemoveRedundantSpaces("sécurité des données"); got == "sécurité des données" {- t.Errorf("test premise broken: RemoveRedundantSpaces no longer mangles accented text (%q)", got)- }}// TestRerankByModelFallsBackToTokensWithoutNaturalText covers chunks with