Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
fix(api): keep email-like text in Markdown link URLs when removing urls and emails (#43712)
Muneeb Ullah358ff7883857
diff --git a/api/core/rag/cleaner/clean_processor.py b/api/core/rag/cleaner/clean_processor.pyindex 452251584e6..6cefdcd4986 100644--- a/api/core/rag/cleaner/clean_processor.py+++ b/api/core/rag/cleaner/clean_processor.py@@ -24,10 +24,6 @@ class CleanProcessor:pattern = r"[\t\f\r\x20\u00a0\u1680\u180e\u2000-\u200a\u202f\u205f\u3000]{2,}"text = re.sub(pattern, " ", text)elif pre_processing_rule["id"] == "remove_urls_emails" and pre_processing_rule["enabled"] is True:- # Remove email- pattern = r"([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)"- text = re.sub(pattern, "", text)-# Remove URL but keep Markdown image URLs and link URLs# Replace the ENTIRE markdown link/image with a single placeholder to protect# the link text (which might also be a URL) from being removed@@ -55,6 +51,10 @@ class CleanProcessor:# Then protect markdown imagestext = re.sub(markdown_image_pattern, replace_image_with_placeholder, text)+ # Remove email (after protection so Markdown link/image URLs are left intact)+ pattern = r"([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)"+ text = re.sub(pattern, "", text)+# Now remove all remaining URLsurl_pattern = r"https?://\S+"text = re.sub(url_pattern, "", text)diff --git a/api/tests/unit_tests/core/rag/cleaner/test_clean_processor.py b/api/tests/unit_tests/core/rag/cleaner/test_clean_processor.pyindex 2155342ab98..e173cf9f411 100644--- a/api/tests/unit_tests/core/rag/cleaner/test_clean_processor.py+++ b/api/tests/unit_tests/core/rag/cleaner/test_clean_processor.py@@ -132,6 +132,18 @@ class TestCleanProcessor:expected = "Email: , link: [Click](https://site.com)"assert CleanProcessor.clean(text, process_rule) == expected+ def test_clean_remove_urls_emails_keeps_email_like_text_in_markdown_urls(self):+ """Email-like text inside protected Markdown link/image URLs must not be stripped."""+ process_rule = {"rules": {"pre_processing_rules": [{"id": "remove_urls_emails", "enabled": True}]}}++ text = "[cal](https://x.io/[email protected]&z=1) and "+ assert CleanProcessor.clean(text, process_rule) == text++ # Emails in plain prose are still removed+ assert CleanProcessor.clean("mail [email protected] [cal](https://x.io/[email protected])", process_rule) == (+ "mail [cal](https://x.io/[email protected])"+ )+def test_clean_remove_urls_emails_disabled(self):"""Test remove_urls_emails rule when disabled."""process_rule = {"rules": {"pre_processing_rules": [{"id": "remove_urls_emails", "enabled": False}]}}