Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Port PR17015 to GO (#19969)

qinling02104c618856ba93
diff --git a/internal/common/string_utils.go b/internal/common/string_utils.go
new file mode 100644
index 0000000000..01fa862647
--- /dev/null
+++ b/internal/common/string_utils.go
@@ -0,0 +1,50 @@
+package common
+
+import (
+ "regexp"
+ "strings"
+)
+
+// Redundant-space cleanup regexes. Mirrors
+// common.string_utils.remove_redundant_spaces (string_utils.py:20-46).
+// Pass 1: drop spaces after a "left boundary" character (parens, <, >).
+// Pass 2: drop spaces before a "right boundary" character (parens, !).
+//
+// The word-character class is Unicode-aware (`\p{L}`, `\p{N}`, `_`), matching
+// Python's `\w`. An ASCII-only `a-z0-9` puts every non-Latin letter into the
+// negated "punctuation" set, which deletes the space between two words:
+// "привет мир" -> "приветмир". Letter case needs no flag because `\p{L}`
+// covers both cases — that is what Python's `re.IGNORECASE` does for `a-z`.
+//
+// The non-space class is a literal space, like Python's `[^ ]`: a tab or
+// newline is a boundary character there, not whitespace to skip.
+var (
+ redundantSpacePass1Re = regexp.MustCompile(`([^\p{L}\p{N}_.,)>]) +([^ ])`)
+ redundantSpacePass2Re = regexp.MustCompile(`([^ ]) +([^\p{L}\p{N}_.,(<])`)
+)
+
+// RemoveRedundantSpaces removes redundant spaces around punctuation marks
+// while preserving the spaces between words.
+func RemoveRedundantSpaces(s string) string {
+ s = redundantSpacePass1Re.ReplaceAllString(s, "$1$2")
+ s = redundantSpacePass2Re.ReplaceAllString(s, "$1$2")
+ return s
+}
+
+// Markdown-fence recognizers. Mirrors
+// common.string_utils.clean_markdown_block (string_utils.py:49-...).
+// Matches Python without re.MULTILINE so ^/$ anchor only the whole text.
+var (
+ markdownFenceOpenRe = regexp.MustCompile(`^\s*` + "```" + `markdown\s*\n?`)
+ markdownFenceCloseRe = regexp.MustCompile(`\n?\s*` + "```" + `\s*$`)
+)
+
+// CleanMarkdownBlock removes a closing fence only after recognizing an opening
+// markdown wrapper; anything else is returned trimmed and otherwise untouched.
+func CleanMarkdownBlock(s string) string {
+ if markdownFenceOpenRe.MatchString(s) {
+ s = markdownFenceOpenRe.ReplaceAllString(s, "")
+ s = markdownFenceCloseRe.ReplaceAllString(s, "")
+ }
+ return strings.TrimSpace(s)
+}
diff --git a/internal/common/string_utils_test.go b/internal/common/string_utils_test.go
new file mode 100644
index 0000000000..86fef7b395
--- /dev/null
+++ b/internal/common/string_utils_test.go
@@ -0,0 +1,148 @@
+package common
+
+import "testing"
+
+// TestRemoveRedundantSpaces mirrors common.string_utils.remove_redundant_spaces.
+func TestRemoveRedundantSpaces(t *testing.T) {
+ cases := []struct {
+ name string
+ in string
+ want string
+ }{
+ // Both passes run sequentially — pass 1 strips space after `(`,
+ // pass 2 strips space before `)`, so both go.
+ {"pass1+pass2 on ( world )", "hello ( world )", "hello (world)"},
+ // Pass 2 strips space before `!`.
+ {"pass2: space before !", "world !", "world!"},
+ // Comma is not a boundary in pass 2 (it's in the negated set
+ // along with `<` and `(`), so no change.
+ {"comma not a boundary", "a , b", "a , b"},
+ {"no match", "foo bar", "foo bar"},
+ {"empty", "", ""},
+ {"digit not a boundary", "abc 123", "abc 123"},
+ {"left paren kept (no following space)", "(abc)", "(abc)"},
+ // Uppercase letters are word characters too: the Python port compiles
+ // with re.IGNORECASE, so "Hello World" must not become "HelloWorld".
+ {"uppercase is a word char", "Hello World", "Hello World"},
+ {"uppercase initials", "A B", "A B"},
+ // Regression: non-Latin letters must not fall into the negated
+ // "punctuation" set, otherwise the space between two words is deleted
+ // ("привет мир" -> "приветмир").
+ {"russian", "привет мир", "привет мир"},
+ {"greek", "Γειά σου κόσμε", "Γειά σου κόσμε"},
+ {"korean", "안녕 세계", "안녕 세계"},
+ {"arabic", "مرحبا بالعالم", "مرحبا بالعالم"},
+ {"chinese", "中文 测试 句子", "中文 测试 句子"},
+ // Punctuation cleanup still applies to non-Latin text.
+ {"accented, space before !", "sécurité des données !", "sécurité des données!"},
+ {"space before punctuation is still dropped", "привет !", "привет!"},
+ {"full-width parens", "( 中文 )", "(中文)"},
+ // The non-space class is a literal space, like Python's `[^ ]`: a
+ // newline or tab is a boundary character, not whitespace to skip.
+ {"newline is a boundary", "hello \nworld", "hello\nworld"},
+ {"tab is a boundary", "foo\t bar", "foo\tbar"},
+ {"CRLF run collapses", "x\r\n y", "x\r\ny"},
+ // `<` is a left boundary: the space after it goes, the one before stays.
+ {"space after < is dropped, before kept", "a < b", "a <b"},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ if got := RemoveRedundantSpaces(tc.in); got != tc.want {
+ t.Errorf("RemoveRedundantSpaces(%q) = %q, want %q", tc.in, got, tc.want)
+ }
+ })
+ }
+}
+
+// TestCleanMarkdownBlock mirrors common.string_utils.clean_markdown_block.
+func TestCleanMarkdownBlock(t *testing.T) {
+ cases := []struct {
+ name string
+ input string
+ want string
+ }{
+ {
+ name: "standard markdown block",
+ input: "```markdown\nHello world\n```",
+ want: "Hello world",
+ },
+ {
+ name: "windows CRLF endings",
+ input: "```markdown\r\nHello world\r\n```",
+ want: "Hello world",
+ },
+ {
+ name: "nested code blocks preserved",
+ input: "```markdown\nText with ```nested``` blocks\n```",
+ want: "Text with ```nested``` blocks",
+ },
+ {
+ name: "mixed whitespace tabs",
+ input: "\t```markdown\t\n\tContent with tabs\n\t```\t",
+ want: "Content with tabs",
+ },
+ {
+ name: "empty string",
+ input: "",
+ want: "",
+ },
+ {
+ name: "only markdown tags",
+ input: "```markdown```",
+ want: "",
+ },
+ {
+ name: "unwrapped text unchanged",
+ input: "Just plain text without markdown block",
+ want: "Just plain text without markdown block",
+ },
+ {
+ name: "unwrapped python block",
+ input: "```python\nprint('hello')\n```",
+ want: "```python\nprint('hello')\n```",
+ },
+ {
+ name: "unwrapped JSON block",
+ input: "```json\n{\"value\": 1}\n```",
+ want: "```json\n{\"value\": 1}\n```",
+ },
+ {
+ name: "unlabeled code block",
+ input: "```\ncode without a language\n```",
+ want: "```\ncode without a language\n```",
+ },
+ {
+ name: "prose ending with code block",
+ input: "Example:\n```python\nprint('hello')\n```",
+ want: "Example:\n```python\nprint('hello')\n```",
+ },
+ {
+ name: "multiple unwrapped code blocks",
+ input: "```python\nfirst()\n```\n\n```python\nsecond()\n```",
+ want: "```python\nfirst()\n```\n\n```python\nsecond()\n```",
+ },
+ {
+ name: "unwrapped CRLF block",
+ input: " ```python\r\nprint('hello')\r\n``` ",
+ want: "```python\r\nprint('hello')\r\n```",
+ },
+ {
+ name: "closing fence without markdown opener",
+ input: "Unopened block\n```",
+ want: "Unopened block\n```",
+ },
+ {
+ name: "markdown wrapper around code example",
+ input: "```markdown\nExample:\n```python\nprint('hello')\n```\n```",
+ want: "Example:\n```python\nprint('hello')\n```",
+ },
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ if got := CleanMarkdownBlock(tc.input); got != tc.want {
+ t.Errorf("CleanMarkdownBlock(%q) = %q, want %q", tc.input, got, tc.want)
+ }
+ })
+ }
+}
diff --git a/internal/ingestion/component/vision_enhancement.go b/internal/ingestion/component/vision_enhancement.go
index 7064e1c3d6..f237997713 100644
--- a/internal/ingestion/component/vision_enhancement.go
+++ b/internal/ingestion/component/vision_enhancement.go
@@ -29,11 +29,11 @@ import (
"log/slog"
"os"
"path/filepath"
- "regexp"
"strings"
"sync"
"time"
+ "ragflow/internal/common"
pdflayout "ragflow/internal/deepdoc/parser/pdf/layout"
"ragflow/internal/entity"
modelModule "ragflow/internal/entity/models"
@@ -90,23 +90,6 @@ func (s *promptDirState) resolve(requestedRoot string) (string, error) {
return s.root, nil
}
-// cleanMarkdownBlock mirrors Python common/string_utils.py:clean_markdown_block
-// and removes a closing fence only after recognizing an opening markdown wrapper.
-//
-// Matches Python without re.MULTILINE so ^/$ anchor only the whole text.
-var (
- reMarkdownOpen = regexp.MustCompile(`^\s*` + "```" + `markdown\s*\n?`)
- reMarkdownClose = regexp.MustCompile(`\n?\s*` + "```" + `\s*$`)
-)
-
-func cleanMarkdownBlock(s string) string {
- if reMarkdownOpen.MatchString(s) {
- s = reMarkdownOpen.ReplaceAllString(s, "")
- s = reMarkdownClose.ReplaceAllString(s, "")
- }
- return strings.TrimSpace(s)
-}
-
// isUsableVisionImage reports whether raw carries a valid image data URI or a
// base64 payload that can be sent to a vision model.
func isUsableVisionImage(raw string) bool {
@@ -388,7 +371,7 @@ func extractVisionAnswer(resp *modelModule.ChatResponse) string {
if resp == nil || resp.Answer == nil {
return ""
}
- return cleanMarkdownBlock(*resp.Answer)
+ return common.CleanMarkdownBlock(*resp.Answer)
}
func defaultVisionChatInvoker(
diff --git a/internal/ingestion/component/vision_enhancement_test.go b/internal/ingestion/component/vision_enhancement_test.go
index 0b36fdb44a..cb68846f30 100644
--- a/internal/ingestion/component/vision_enhancement_test.go
+++ b/internal/ingestion/component/vision_enhancement_test.go
@@ -476,7 +476,8 @@ func TestVisionEnhancement_MoreThanConcurrencyItems(t *testing.T) {
}
// TestVisionEnhancement_PlainTextResponseNotTruncated verifies that a plain-text
-// (no fences) VLM response is returned verbatim — cleanMarkdownBlock must not truncate it.
+// (no fences) VLM response is returned verbatim — common.CleanMarkdownBlock
+// must not truncate it.
func TestVisionEnhancement_PlainTextResponseNotTruncated(t *testing.T) {
plain := "A pipeline diagram showing three stages."
swapVisionGlobals(t, fakeResolver,
@@ -709,7 +710,10 @@ func TestExtractVisionAnswer_CleansMarkdownBlock(t *testing.T) {
}
}
-func TestCleanMarkdownBlock_EdgeCases(t *testing.T) {
+// TestExtractVisionAnswer_EdgeCases drives the shared
+// common.CleanMarkdownBlock through the vision-answer path; the cleanup
+// function itself is pinned in common.TestCleanMarkdownBlock.
+func TestExtractVisionAnswer_EdgeCases(t *testing.T) {
cases := []struct {
name string
input string
@@ -794,10 +798,6 @@ func TestCleanMarkdownBlock_EdgeCases(t *testing.T) {
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
- got := cleanMarkdownBlock(tc.input)
- if got != tc.want {
- t.Errorf("cleanMarkdownBlock(%q) = %q, want %q", tc.input, got, tc.want)
- }
resp := &modelModule.ChatResponse{Answer: &tc.input}
if got := extractVisionAnswer(resp); got != tc.want {
t.Errorf("extractVisionAnswer(%q) = %q, want %q", tc.input, got, tc.want)
diff --git a/internal/service/chat_pipeline.go b/internal/service/chat_pipeline.go
index f4a8c6f686..6ecc10ac1e 100644
--- a/internal/service/chat_pipeline.go
+++ b/internal/service/chat_pipeline.go
@@ -4363,23 +4363,6 @@ func normalizeSQL(s string) string {
// Python parity helpers (dialog_service.py:56-59, 1238-1309, 1321-1365)
// -----------------------------------------------------------------------
-// Redundant-space cleanup regexes. Mirrors
-// common.string_utils.remove_redundant_spaces (string_utils.py:20-46).
-// Pass 1: drop spaces after a "left boundary" character (parens, <, >).
-// Pass 2: drop spaces before a "right boundary" character (parens, !).
-var (
- redundantSpacePass1Re = regexp.MustCompile(`([^a-z0-9.,)>\x{ff08}]) +([^ ])`) // left boundary + space + non-space
- redundantSpacePass2Re = regexp.MustCompile(`([^ ]) +([^a-z0-9.,(<])`) // non-space + space + right boundary
-)
-
-// removeRedundantSpaces ports common.string_utils.remove_redundant_spaces.
-// Two-pass regex cleanup; both passes use case-insensitive matching.
-func removeRedundantSpaces(s string) string {
- s = redundantSpacePass1Re.ReplaceAllString(s, "$1$2")
- s = redundantSpacePass2Re.ReplaceAllString(s, "$1$2")
- return s
-}
-
// ISO timestamp stripping regex. Mirrors the cleanup at
// dialog_service.py:1309. Matches `T13:24:55|` or `T13:24:55.123Z|`.
var isoTimestampCellRe = regexp.MustCompile(`T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+Z)?\|`)
@@ -4468,7 +4451,7 @@ func chunkKBIDForDoc(rowDict map[string]interface{}, kbIDs []string, docID inter
func cleanCellValue(v interface{}) string {
s := fmt.Sprintf("%v", v)
s = strings.ReplaceAll(s, "None", " ")
- return removeRedundantSpaces(s)
+ return common.RemoveRedundantSpaces(s)
}
// extractSourceColumnIndexes returns, for a set of SQL result rows,
diff --git a/internal/service/chat_pipeline_test.go b/internal/service/chat_pipeline_test.go
index c64e5ae6b4..8e44a8f9bd 100644
--- a/internal/service/chat_pipeline_test.go
+++ b/internal/service/chat_pipeline_test.go
@@ -924,35 +924,6 @@ func TestShouldUseWebSearch_RequiresTavilyAndTruthyInternet(t *testing.T) {
// --- P5 SQL retrieval parity helpers (Python use_sql alignment) ---
-// TestRemoveRedundantSpaces mirrors common.string_utils.remove_redundant_spaces.
-func TestRemoveRedundantSpaces(t *testing.T) {
- cases := []struct {
- name string
- in string
- want string
- }{
- // Both passes run sequentially — pass 1 strips space after `(`,
- // pass 2 strips space before `)`, so both go.
- {"pass1+pass2 on ( world )", "hello ( world )", "hello (world)"},
- // Pass 2 strips space before `!`.
- {"pass2: space before !", "world !", "world!"},
- // Comma is not a boundary in pass 2 (it's in the negated set
- // along with `<` and `(`), so no change.
- {"comma not a boundary", "a , b", "a , b"},
- {"no match", "foo bar", "foo bar"},
- {"empty", "", ""},
- {"digit not a boundary", "abc 123", "abc 123"},
- {"left paren kept (no following space)", "(abc)", "(abc)"},
- }
- for _, tc := range cases {
- t.Run(tc.name, func(t *testing.T) {
- if got := removeRedundantSpaces(tc.in); got != tc.want {
- t.Errorf("removeRedundantSpaces(%q) = %q, want %q", tc.in, got, tc.want)
- }
- })
- }
-}
-
// TestStripISOTimestamps verifies the dialog_service.py:1309 cleanup.
// The pattern `T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+Z)?\|` strips the
// timestamp + trailing pipe; the leading pipe/space is preserved (the
diff --git a/internal/service/nlp/reranker.go b/internal/service/nlp/reranker.go
index 4678ed1c35..3bf8a54f99 100644
--- a/internal/service/nlp/reranker.go
+++ b/internal/service/nlp/reranker.go
@@ -18,7 +18,6 @@ import (
"context"
"encoding/json"
"math"
- "regexp"
"sort"
"strconv"
"strings"
@@ -152,13 +151,12 @@ func RerankByModel(
// tokenized content_ltks. Neural rerankers score stemmed / accent-split
// tokens far lower, which collapses relevance scores and forces an
// artificially low similarity_threshold. The natural text is passed
- // as-is: RemoveRedundantSpaces is ASCII-oriented and mangles
- // multilingual text ("sécurité des données" -> "sécuritédes données"),
- // so it is only applied to the tokenized fallback used when
- // content_with_weight is absent. Mirrors rag/nlp/search.py.
+ // as-is; common.RemoveRedundantSpaces is only applied to the tokenized
+ // fallback used when content_with_weight is absent, where it drops the
+ // space before punctuation tokens. Mirrors rag/nlp/search.py.
docText := extractNaturalText(chunk)
if docText == "" {
- docText = RemoveRedundantSpaces(strings.Join(tks, " "))
+ docText = common.RemoveRedundantSpaces(strings.Join(tks, " "))
}
docs = append(docs, docText)
}
@@ -575,7 +573,7 @@ func extractTitleTokens(fields map[string]interface{}) []string {
if !ok {
return []string{}
}
- // NOTE: Do NOT call RemoveRedundantSpaces here - it removes spaces between Chinese chars
+ // title_tks is a space-separated token list: split it, do not clean it as prose.
var result []string
for t := range strings.FieldsSeq(v) {
if t != "" {
@@ -652,21 +650,6 @@ func cosineSimilarity(a, b []float64) float64 {
return dot / (common.PySqrt(normA) * common.PySqrt(normB))
}
-// RemoveRedundantSpaces removes redundant spaces from text
-// First pass: remove spaces after left-boundary characters
-// Second pass: remove spaces before right-boundary characters
-func RemoveRedundantSpaces(s string) string {
- // First pass: remove spaces after left-boundary characters (opening brackets, etc.)
- // e.g., "( text" -> "(text", "【 text" -> "【text"
- s = regexp.MustCompile(`([^\sa-z0-9.,\)>]) +([^\s])`).ReplaceAllString(s, "$1$2")
-
- // Second pass: remove spaces before right-boundary characters (closing brackets, punctuation)
- // e.g., "text !" -> "text!"
- s = regexp.MustCompile(`([^\s]) +([^\sa-z0-9.,\(])`).ReplaceAllString(s, "$1$2")
-
- return s
-}
-
// parseFloat parses a string to float64
func parseFloat(s string) (float64, error) {
return strconv.ParseFloat(strings.TrimSpace(s), 64)
diff --git a/internal/service/nlp/reranker_question_tks_test.go b/internal/service/nlp/reranker_question_tks_test.go
index a6a2204396..3fde4ebcf9 100644
--- a/internal/service/nlp/reranker_question_tks_test.go
+++ b/internal/service/nlp/reranker_question_tks_test.go
@@ -130,10 +130,11 @@ func TestRerankByModelPrefersNaturalText(t *testing.T) {
}
}
-// TestRerankByModelNaturalTextKeepsAccentedSpacing guards against
-// RemoveRedundantSpaces being applied to natural text: it treats every
-// non-ASCII letter as punctuation and would collapse "sécurité des" into
-// "sécuritédes".
+// TestRerankByModelNaturalTextKeepsAccentedSpacing guards against the
+// redundant-space cleanup being applied to natural text: an ASCII-only word
+// class treats every non-Latin letter as punctuation and collapses
+// "sécurité des" into "sécuritédes". The cleanup itself is pinned in
+// common.TestRemoveRedundantSpaces.
func TestRerankByModelNaturalTextKeepsAccentedSpacing(t *testing.T) {
chunk := fullChunk()
chunk["content_with_weight"] = "sécurité des données"
@@ -143,9 +144,6 @@ func TestRerankByModelNaturalTextKeepsAccentedSpacing(t *testing.T) {
if doc != "sécurité des données" {
t.Errorf("natural text was altered: got %q", doc)
}
- if got := RemoveRedundantSpaces("sécurité des données"); got == "sécurité des données" {
- t.Errorf("test premise broken: RemoveRedundantSpaces no longer mangles accented text (%q)", got)
- }
}
// TestRerankByModelFallsBackToTokensWithoutNaturalText covers chunks with