Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(rag): stop the document cleaner from stripping valid characters ï, ¿, ¾ (#39215)

Madan kumaraeda37db6871
diff --git a/api/core/indexing_runner.py b/api/core/indexing_runner.py
index 92246b6614c..63e00f48324 100644
--- a/api/core/indexing_runner.py
+++ b/api/core/indexing_runner.py
@@ -519,7 +519,7 @@ class IndexingRunner:
def filter_string(text):
text = re.sub(r"<\|", "<", text)
text = re.sub(r"\|>", ">", text)
- text = re.sub(r"[\x00-\x08\x0B\x0C\x0E-\x1F\x7F\xEF\xBF\xBE]", "", text)
+ text = re.sub(r"[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]", "", text)
# Unicode U+FFFE
text = re.sub("\ufffe", "", text)
return text
diff --git a/api/core/rag/cleaner/clean_processor.py b/api/core/rag/cleaner/clean_processor.py
index 790253053de..452251584e6 100644
--- a/api/core/rag/cleaner/clean_processor.py
+++ b/api/core/rag/cleaner/clean_processor.py
@@ -9,7 +9,7 @@ class CleanProcessor:
# remove invalid symbol
text = re.sub(r"<\|", "<", text)
text = re.sub(r"\|>", ">", text)
- text = re.sub(r"[\x00-\x08\x0B\x0C\x0E-\x1F\x7F\xEF\xBF\xBE]", "", text)
+ text = re.sub(r"[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]", "", text)
# Unicode U+FFFE
text = re.sub("\ufffe", "", text)
diff --git a/api/tests/unit_tests/core/rag/cleaner/test_clean_processor.py b/api/tests/unit_tests/core/rag/cleaner/test_clean_processor.py
index c7a4265a954..2155342ab98 100644
--- a/api/tests/unit_tests/core/rag/cleaner/test_clean_processor.py
+++ b/api/tests/unit_tests/core/rag/cleaner/test_clean_processor.py
@@ -22,6 +22,23 @@ class TestCleanProcessor:
expected = "normalpadding"
assert CleanProcessor.clean(text_with_ufffe, None) == expected
+ def test_clean_preserves_valid_extended_characters(self):
+ """Default cleaning must not strip valid printable characters.
+
+ The invalid-symbol filter used to include the UTF-8 bytes of U+FFFE
+ (0xEF 0xBF 0xBE) inside a character class. On a decoded string those
+ bytes are the code points U+00EF, U+00BF and U+00BE, i.e. the valid
+ characters 'ï', '¿' and '¾', so words like "naïve" and Spanish
+ questions like "¿Cómo?" were being silently corrupted on ingest.
+ """
+ assert CleanProcessor.clean("naïve", None) == "naïve"
+ assert CleanProcessor.clean("¿Cómo estás?", None) == "¿Cómo estás?"
+ assert CleanProcessor.clean("¾ cup sugar", None) == "¾ cup sugar"
+ assert CleanProcessor.clean("￾", None) == "￾"
+
+ # The U+FFFE noncharacter is still stripped by its dedicated substitution.
+ assert CleanProcessor.clean("keep\ufffedrop", None) == "keepdrop"
+
def test_clean_with_none_process_rule(self):
"""Test cleaning with None process_rule - only default cleaning applied."""
text = "Hello<|World\x00"
diff --git a/api/tests/unit_tests/core/rag/indexing/test_indexing_runner.py b/api/tests/unit_tests/core/rag/indexing/test_indexing_runner.py
index 5307da6d343..ef1b38f49d7 100644
--- a/api/tests/unit_tests/core/rag/indexing/test_indexing_runner.py
+++ b/api/tests/unit_tests/core/rag/indexing/test_indexing_runner.py
@@ -1372,6 +1372,19 @@ class TestIndexingRunnerDocumentCleaning:
assert "\ufffe" not in result
assert "Text with" in result
+ def test_filter_string_preserves_valid_extended_characters(self):
+ """filter_string must keep valid printable characters like 'ï', '¿', '¾'."""
+ # Arrange
+ text = "naïve ¿Cómo? ¾ done"
+
+ # Act
+ result = IndexingRunner.filter_string(text)
+
+ # Assert
+ assert result == text
+ # The U+FFFE noncharacter is still stripped.
+ assert IndexingRunner.filter_string("keep\ufffedrop") == "keepdrop"
+
class TestIndexingRunnerSplitter:
"""Unit tests for text splitter configuration.