Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
fix(rag): stop the document cleaner from stripping valid characters ï, ¿, ¾ (#39215)
Madan kumaraeda37db6871
diff --git a/api/core/indexing_runner.py b/api/core/indexing_runner.pyindex 92246b6614c..63e00f48324 100644--- a/api/core/indexing_runner.py+++ b/api/core/indexing_runner.py@@ -519,7 +519,7 @@ class IndexingRunner:def filter_string(text):text = re.sub(r"<\|", "<", text)text = re.sub(r"\|>", ">", text)- text = re.sub(r"[\x00-\x08\x0B\x0C\x0E-\x1F\x7F\xEF\xBF\xBE]", "", text)+ text = re.sub(r"[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]", "", text)# Unicode U+FFFEtext = re.sub("\ufffe", "", text)return textdiff --git a/api/core/rag/cleaner/clean_processor.py b/api/core/rag/cleaner/clean_processor.pyindex 790253053de..452251584e6 100644--- a/api/core/rag/cleaner/clean_processor.py+++ b/api/core/rag/cleaner/clean_processor.py@@ -9,7 +9,7 @@ class CleanProcessor:# remove invalid symboltext = re.sub(r"<\|", "<", text)text = re.sub(r"\|>", ">", text)- text = re.sub(r"[\x00-\x08\x0B\x0C\x0E-\x1F\x7F\xEF\xBF\xBE]", "", text)+ text = re.sub(r"[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]", "", text)# Unicode U+FFFEtext = re.sub("\ufffe", "", text)diff --git a/api/tests/unit_tests/core/rag/cleaner/test_clean_processor.py b/api/tests/unit_tests/core/rag/cleaner/test_clean_processor.pyindex c7a4265a954..2155342ab98 100644--- a/api/tests/unit_tests/core/rag/cleaner/test_clean_processor.py+++ b/api/tests/unit_tests/core/rag/cleaner/test_clean_processor.py@@ -22,6 +22,23 @@ class TestCleanProcessor:expected = "normalpadding"assert CleanProcessor.clean(text_with_ufffe, None) == expected+ def test_clean_preserves_valid_extended_characters(self):+ """Default cleaning must not strip valid printable characters.++ The invalid-symbol filter used to include the UTF-8 bytes of U+FFFE+ (0xEF 0xBF 0xBE) inside a character class. On a decoded string those+ bytes are the code points U+00EF, U+00BF and U+00BE, i.e. the valid+ characters 'ï', '¿' and '¾', so words like "naïve" and Spanish+ questions like "¿Cómo?" were being silently corrupted on ingest.+ """+ assert CleanProcessor.clean("naïve", None) == "naïve"+ assert CleanProcessor.clean("¿Cómo estás?", None) == "¿Cómo estás?"+ assert CleanProcessor.clean("¾ cup sugar", None) == "¾ cup sugar"+ assert CleanProcessor.clean("￾", None) == "￾"++ # The U+FFFE noncharacter is still stripped by its dedicated substitution.+ assert CleanProcessor.clean("keep\ufffedrop", None) == "keepdrop"+def test_clean_with_none_process_rule(self):"""Test cleaning with None process_rule - only default cleaning applied."""text = "Hello<|World\x00"diff --git a/api/tests/unit_tests/core/rag/indexing/test_indexing_runner.py b/api/tests/unit_tests/core/rag/indexing/test_indexing_runner.pyindex 5307da6d343..ef1b38f49d7 100644--- a/api/tests/unit_tests/core/rag/indexing/test_indexing_runner.py+++ b/api/tests/unit_tests/core/rag/indexing/test_indexing_runner.py@@ -1372,6 +1372,19 @@ class TestIndexingRunnerDocumentCleaning:assert "\ufffe" not in resultassert "Text with" in result+ def test_filter_string_preserves_valid_extended_characters(self):+ """filter_string must keep valid printable characters like 'ï', '¿', '¾'."""+ # Arrange+ text = "naïve ¿Cómo? ¾ done"++ # Act+ result = IndexingRunner.filter_string(text)++ # Assert+ assert result == text+ # The U+FFFE noncharacter is still stripped.+ assert IndexingRunner.filter_string("keep\ufffedrop") == "keepdrop"+class TestIndexingRunnerSplitter:"""Unit tests for text splitter configuration.