RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(string_utils): keep spaces between non-Latin words (#17015)
Andrew Chenfc7a0cbca351
diff --git a/common/string_utils.py b/common/string_utils.pyindex fdc513af28..2bf58799c3 100644--- a/common/string_utils.py+++ b/common/string_utils.py@@ -32,18 +32,20 @@ def remove_redundant_spaces(txt: str):str: Text with redundant spaces removed"""# First pass: Remove spaces after left-boundary characters- # Matches: [non-alphanumeric-and-specific-right-punctuation] + [non-space]- # Removes spaces after characters like '(', '<', and other non-alphanumeric chars+ # Matches: [non-word-char-and-specific-right-punctuation] + [non-space]+ # Removes spaces after characters like '(', '<', and other non-word chars.+ # `\w` (not `a-z0-9`) so that non-ASCII letters count as word chars and the+ # space between two non-Latin words is preserved.# Examples:# "( test" → "(test"- txt = re.sub(r"([^a-z0-9.,\)>]) +([^ ])", r"\1\2", txt, flags=re.IGNORECASE)+ txt = re.sub(r"([^\w.,\)>]) +([^ ])", r"\1\2", txt, flags=re.IGNORECASE)# Second pass: Remove spaces before right-boundary characters- # Matches: [non-space] + [non-alphanumeric-and-specific-left-punctuation]- # Removes spaces before characters like non-')', non-',', non-'.', and non-alphanumeric chars+ # Matches: [non-space] + [non-word-char-and-specific-left-punctuation]+ # Removes spaces before characters like non-')', non-',', non-'.', and non-word chars# Examples:# "world !" → "world!"- return re.sub(r"([^ ]) +([^a-z0-9.,\(<])", r"\1\2", txt, flags=re.IGNORECASE)+ return re.sub(r"([^ ]) +([^\w.,\(<])", r"\1\2", txt, flags=re.IGNORECASE)def clean_markdown_block(text):diff --git a/test/unit_test/common/test_string_utils.py b/test/unit_test/common/test_string_utils.pyindex cb682a53cc..26a6cc93eb 100644--- a/test/unit_test/common/test_string_utils.py+++ b/test/unit_test/common/test_string_utils.py@@ -58,6 +58,24 @@ class TestRemoveRedundantSpaces:expected = "This should remain unchanged"assert remove_redundant_spaces(input_text) == expected+ @pytest.mark.parametrize(+ "text",+ [+ "привет мир", # Russian+ "Γειά σου κόσμε", # Greek+ "안녕 세계", # Korean+ "مرحبا بالعالم", # Arabic+ ],+ )+ def test_keep_spaces_between_non_latin_words(self, text):+ """Regression: spaces between non-Latin words must be preserved.++ The whitelist used ``a-z0-9`` (ASCII only), so a non-ASCII letter fell+ into the negated class and was treated as punctuation, deleting the+ space between two words (e.g. "привет мир" -> "приветмир").+ """+ assert remove_redundant_spaces(text) == text+@pytest.mark.skip(reason="Failed")def test_mixed_punctuation(self):"""Test mixed punctuation scenarios"""