Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(api): keep email-like text in Markdown link URLs when removing urls and emails (#43712)

Muneeb Ullah358ff7883857
diff --git a/api/core/rag/cleaner/clean_processor.py b/api/core/rag/cleaner/clean_processor.py
index 452251584e6..6cefdcd4986 100644
--- a/api/core/rag/cleaner/clean_processor.py
+++ b/api/core/rag/cleaner/clean_processor.py
@@ -24,10 +24,6 @@ class CleanProcessor:
pattern = r"[\t\f\r\x20\u00a0\u1680\u180e\u2000-\u200a\u202f\u205f\u3000]{2,}"
text = re.sub(pattern, " ", text)
elif pre_processing_rule["id"] == "remove_urls_emails" and pre_processing_rule["enabled"] is True:
- # Remove email
- pattern = r"([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)"
- text = re.sub(pattern, "", text)
-
# Remove URL but keep Markdown image URLs and link URLs
# Replace the ENTIRE markdown link/image with a single placeholder to protect
# the link text (which might also be a URL) from being removed
@@ -55,6 +51,10 @@ class CleanProcessor:
# Then protect markdown images
text = re.sub(markdown_image_pattern, replace_image_with_placeholder, text)
+ # Remove email (after protection so Markdown link/image URLs are left intact)
+ pattern = r"([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)"
+ text = re.sub(pattern, "", text)
+
# Now remove all remaining URLs
url_pattern = r"https?://\S+"
text = re.sub(url_pattern, "", text)
diff --git a/api/tests/unit_tests/core/rag/cleaner/test_clean_processor.py b/api/tests/unit_tests/core/rag/cleaner/test_clean_processor.py
index 2155342ab98..e173cf9f411 100644
--- a/api/tests/unit_tests/core/rag/cleaner/test_clean_processor.py
+++ b/api/tests/unit_tests/core/rag/cleaner/test_clean_processor.py
@@ -132,6 +132,18 @@ class TestCleanProcessor:
expected = "Email: , link: [Click](https://site.com)"
assert CleanProcessor.clean(text, process_rule) == expected
+ def test_clean_remove_urls_emails_keeps_email_like_text_in_markdown_urls(self):
+ """Email-like text inside protected Markdown link/image URLs must not be stripped."""
+ process_rule = {"rules": {"pre_processing_rules": [{"id": "remove_urls_emails", "enabled": True}]}}
+
+ text = "[cal](https://x.io/[email protected]&z=1) and ![img](https://x.io/[email protected])"
+ assert CleanProcessor.clean(text, process_rule) == text
+
+ # Emails in plain prose are still removed
+ assert CleanProcessor.clean("mail [email protected] [cal](https://x.io/[email protected])", process_rule) == (
+ "mail [cal](https://x.io/[email protected])"
+ )
+
def test_clean_remove_urls_emails_disabled(self):
"""Test remove_urls_emails rule when disabled."""
process_rule = {"rules": {"pre_processing_rules": [{"id": "remove_urls_emails", "enabled": False}]}}