Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

feat(api): normalize additional email providers (#42112)

林玮 (Jade Lin)cd61738f3350
diff --git a/api/migrations/versions/2026_09_10_1200-d8e4a6b1c902_expand_normalized_email_providers.py b/api/migrations/versions/2026_09_10_1200-d8e4a6b1c902_expand_normalized_email_providers.py
new file mode 100644
index 00000000000..6ccfd865330
--- /dev/null
+++ b/api/migrations/versions/2026_09_10_1200-d8e4a6b1c902_expand_normalized_email_providers.py
@@ -0,0 +1,74 @@
+"""Expand account email normalization to Microsoft, Apple, and Proton.
+
+Keep these rules self-contained so future runtime policy changes cannot alter
+historical migrations. Equivalent existing accounts retain their nonunique
+normalized values and their original email addresses.
+
+Policy sources:
+- Microsoft +site delivery (preserve dots and domains):
+ https://support.microsoft.com/en-us/outlook/send-email-from-a-different-address-in-outlook-com
+- Apple legacy domain association for eligible migrated accounts:
+ https://support.apple.com/en-us/118230
+ +tag delivery was manually verified by linw1995 on 2026-09-10;
+ the official source covers legacy domains, not +tag delivery.
+- Proton +aliases and consumer domains:
+ https://proton.me/support/addresses-and-aliases
+- Proton transparent username characters (dots, underscores, and hyphens):
+ https://proton.me/support/change-username
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "d8e4a6b1c902"
+down_revision = "c3f1a9b2e6d4"
+branch_labels = None
+depends_on = None
+
+_MICROSOFT_DOMAINS = ("outlook.com", "hotmail.com", "live.com", "msn.com")
+_APPLE_DOMAINS = ("icloud.com", "me.com", "mac.com")
+_PROTON_DOMAINS = ("protonmail.com", "proton.me", "pm.me", "protonmail.ch")
+
+
+def _backfill_normalized_emails(*, restore: bool = False) -> None:
+ dialect = op.get_context().dialect.name
+ email = sa.func.lower(sa.column("email"))
+ if dialect == "postgresql":
+ domain = sa.func.split_part(email, "@", 2)
+ local_part = sa.func.split_part(sa.func.split_part(email, "@", 1), "+", 1)
+ elif dialect in {"mysql", "mariadb"}:
+ domain = sa.func.substring_index(email, "@", -1)
+ local_part = sa.func.substring_index(sa.func.substring_index(email, "@", 1), "+", 1)
+ elif dialect == "sqlite":
+ domain = sa.func.substr(email, sa.func.instr(email, "@") + 1)
+ local = sa.func.substr(email, 1, sa.func.instr(email, "@") - 1)
+ tagged_local = sa.cast(local, sa.String) + "+"
+ local_part = sa.func.substr(local, 1, sa.func.instr(tagged_local, "+") - 1)
+ else:
+ raise RuntimeError(f"Unsupported database dialect for normalized email migration: {dialect}")
+
+ normalized_local = sa.case(
+ (
+ domain.in_(_PROTON_DOMAINS),
+ sa.func.replace(sa.func.replace(sa.func.replace(local_part, ".", ""), "_", ""), "-", ""),
+ ),
+ else_=local_part,
+ )
+ normalized_domain = sa.case((domain.in_(_APPLE_DOMAINS), "icloud.com"), else_=domain)
+ normalized_email = sa.cast(normalized_local, sa.String) + "@" + sa.cast(normalized_domain, sa.String)
+ accounts = sa.table("accounts", sa.column("email"), sa.column("normalized_email"))
+ statement = (
+ accounts.update()
+ .where(domain.in_(_MICROSOFT_DOMAINS + _APPLE_DOMAINS + _PROTON_DOMAINS))
+ .values(normalized_email=email if restore else normalized_email)
+ )
+ # Render the fixed policy constants so offline migrations are executable too.
+ op.execute(str(statement.compile(dialect=op.get_context().dialect, compile_kwargs={"literal_binds": True})))
+
+
+def upgrade() -> None:
+ _backfill_normalized_emails()
+
+
+def downgrade() -> None:
+ _backfill_normalized_emails(restore=True)
diff --git a/api/services/account_email.py b/api/services/account_email.py
index 58723b8ce2b..20f1d6523ae 100644
--- a/api/services/account_email.py
+++ b/api/services/account_email.py
@@ -1,11 +1,47 @@
"""Account email normalization rules."""
+# Microsoft documents +site delivery to the original inbox, not dot removal
+# or automatic equivalence between domains:
+# https://support.microsoft.com/en-us/outlook/send-email-from-a-different-address-in-outlook-com
+# Consumer mailbox domains:
+# https://support.microsoft.com/en-us/outlook/premium-features-in-outlook-com-for-microsoft-365-subscribers
+_MICROSOFT_DOMAINS = frozenset({"outlook.com", "hotmail.com", "live.com", "msn.com"})
+# Apple documents matching legacy domains for eligible migrated accounts:
+# https://support.apple.com/en-us/118230
+# +tag delivery was manually verified by linw1995 on 2026-09-10;
+# the official source above covers legacy domains, not +tag delivery.
+_APPLE_DOMAINS = frozenset({"icloud.com", "me.com", "mac.com"})
+# Proton documents automatic +aliases and its four consumer domains:
+# https://proton.me/support/addresses-and-aliases
+_PROTON_DOMAINS = frozenset({"protonmail.com", "proton.me", "pm.me", "protonmail.ch"})
+
def normalize_email(email: str) -> str:
- """Normalize an account email for identity comparisons."""
+ """Normalize a validated account email for new-account collision checks.
+
+ Provider rules apply only to explicitly listed consumer domains. Yahoo's
+ configured disposable addresses cannot be resolved to their owner from the
+ address alone, so they retain the default lowercase-only normalization.
+ Yahoo documents a separately chosen nickname and keyword:
+ https://en-global.help.yahoo.com/kb/SLN36718.html
+ """
normalized_email = email.lower()
local_part, separator, domain = normalized_email.rpartition("@")
if separator and domain in {"gmail.com", "googlemail.com"}:
+ # Dot equivalence applies to consumer Gmail, not Workspace domains:
+ # https://support.google.com/mail/answer/7436150?hl=en
+ # Google documents +tag variants and continued delivery to googlemail:
+ # https://blog.google/products-and-platforms/products/gmail/gmail-inbox-organizing-tips/
+ # https://blog.google/intl/de-de/produkte/smartes-arbeiten/willkommen-bei-gmail-deutschland/
local_part = local_part.split("+", 1)[0].replace(".", "")
return f"{local_part}@gmail.com"
+ if separator and domain in _MICROSOFT_DOMAINS | _APPLE_DOMAINS | _PROTON_DOMAINS:
+ local_part = local_part.split("+", 1)[0]
+ if domain in _APPLE_DOMAINS:
+ domain = "icloud.com"
+ elif domain in _PROTON_DOMAINS:
+ # Proton treats these characters as transparent within a username:
+ # https://proton.me/support/change-username
+ local_part = local_part.replace(".", "").replace("_", "").replace("-", "")
+ return f"{local_part}@{domain}"
return normalized_email
diff --git a/api/tests/unit_tests/migrations/test_expanded_normalized_email_migration.py b/api/tests/unit_tests/migrations/test_expanded_normalized_email_migration.py
new file mode 100644
index 00000000000..710d6d0265f
--- /dev/null
+++ b/api/tests/unit_tests/migrations/test_expanded_normalized_email_migration.py
@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+import importlib.util
+from io import StringIO
+from pathlib import Path
+from types import ModuleType
+
+import pytest
+import sqlalchemy as sa
+from alembic.migration import MigrationContext
+from alembic.operations import Operations
+
+from services.account_email import normalize_email
+
+
+def _load_migration() -> ModuleType:
+ path = (
+ Path(__file__).resolve().parents[3]
+ / "migrations/versions/2026_09_10_1200-d8e4a6b1c902_expand_normalized_email_providers.py"
+ )
+ spec = importlib.util.spec_from_file_location("expand_normalized_email_providers", path)
+ assert spec is not None
+ assert spec.loader is not None
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+def test_upgrade_matches_runtime_preserves_accounts_and_downgrade_restores_values() -> None:
+ domains = [
+ "outlook.com",
+ "hotmail.com",
+ "live.com",
+ "msn.com",
+ "icloud.com",
+ "me.com",
+ "mac.com",
+ "protonmail.com",
+ "proton.me",
+ "pm.me",
+ "protonmail.ch",
+ "yahoo.com",
+ "yahoo.co.uk",
+ "mail.com",
+ "example.org",
+ "sub.proton.me",
+ "outlook.com.example.org",
+ ]
+ emails = [f"{local}@{domain.upper()}" for domain in domains for local in ("First.Last_Name-Test+tag+other", "User")]
+ original_values = [
+ normalize_email(email) if email.lower().endswith(("@gmail.com", "@googlemail.com")) else email.lower()
+ for email in emails
+ ]
+ engine = sa.create_engine("sqlite:///:memory:")
+ metadata = sa.MetaData()
+ accounts = sa.Table(
+ "accounts",
+ metadata,
+ sa.Column("id", sa.Integer, primary_key=True),
+ sa.Column("email", sa.String(255), nullable=False),
+ sa.Column("normalized_email", sa.String(255), nullable=True),
+ sa.Index("account_normalized_email_idx", "normalized_email"),
+ )
+ metadata.create_all(engine)
+ module = _load_migration()
+ with engine.begin() as connection:
+ connection.execute(
+ accounts.insert(),
+ [
+ {"id": index, "email": email, "normalized_email": original_values[index]}
+ for index, email in enumerate(emails)
+ ],
+ )
+ module.__dict__["op"] = Operations(MigrationContext.configure(connection))
+ module.upgrade()
+ rows = connection.execute(sa.select(accounts).order_by(accounts.c.id)).all()
+ assert [row.email for row in rows] == emails
+ assert [row.normalized_email for row in rows] == [normalize_email(email) for email in emails]
+ assert [row.normalized_email for row in rows].count("[email protected]") == 2
+ module.upgrade()
+ assert connection.execute(sa.select(accounts).order_by(accounts.c.id)).all() == rows
+ module.downgrade()
+ assert (
+ connection.scalars(sa.select(accounts.c.normalized_email).order_by(accounts.c.id)).all() == original_values
+ )
+ engine.dispose()
+
+
[email protected]("dialect", ["postgresql", "mysql", "mariadb", "sqlite"])
[email protected]("restore", [False, True])
+def test_migration_supports_offline_sql(dialect: str, restore: bool) -> None:
+ output = StringIO()
+ module = _load_migration()
+ module.__dict__["op"] = Operations(
+ MigrationContext.configure(
+ dialect_name=dialect,
+ opts={"as_sql": True, "output_buffer": output},
+ )
+ )
+ module._backfill_normalized_emails(restore=restore)
+ sql = output.getvalue()
+ assert "UPDATE accounts SET normalized_email=" in sql
+ assert "WHERE" in sql
+ assert "protonmail.ch" in sql
+ assert "gmail.com" not in sql
+ assert "CASE" in sql if not restore else "CASE" not in sql
diff --git a/api/tests/unit_tests/services/test_account_email.py b/api/tests/unit_tests/services/test_account_email.py
index b01aaa8085e..d5c5ccdca1a 100644
--- a/api/tests/unit_tests/services/test_account_email.py
+++ b/api/tests/unit_tests/services/test_account_email.py
@@ -17,3 +17,37 @@ from services.account_email import normalize_email
)
def test_normalize_email(email: str, expected: str) -> None:
assert normalize_email(email) == expected
+
+
[email protected]("domain", ["outlook.com", "hotmail.com", "live.com", "msn.com"])
+def test_microsoft_strips_tags_but_preserves_dots_and_domains(domain: str) -> None:
+ assert normalize_email(f"First.Last+tag+other@{domain.upper()}") == f"first.last@{domain}"
+ assert normalize_email(f"first.last@{domain}") != normalize_email(f"firstlast@{domain}")
+
+
[email protected]("domain", ["icloud.com", "me.com", "mac.com"])
+def test_apple_normalizes_legacy_domains_and_tags(domain: str) -> None:
+ assert normalize_email(f"First.Last+tag@{domain.upper()}") == "[email protected]"
+ assert normalize_email(f"first.last@{domain}") != normalize_email(f"firstlast@{domain}")
+
+
[email protected]("domain", ["protonmail.com", "proton.me", "pm.me", "protonmail.ch"])
+def test_proton_strips_transparent_characters_and_tags_but_preserves_domain(domain: str) -> None:
+ assert normalize_email(f"First.Last_Name-Test+tag+other@{domain.upper()}") == f"firstlastnametest@{domain}"
+
+
+ "domain",
+ ["yahoo.com", "yahoo.co.uk", "ymail.com", "mail.com", "example.com", "sub.gmail.com", "outlook.com.example.org"],
+)
+def test_other_domains_preserve_local_part(domain: str) -> None:
+ assert normalize_email(f"First.Last_Name-Test+tag@{domain.upper()}") == f"first.last_name-test+tag@{domain}"
+
+
+ "email",
+)
+def test_normalization_is_idempotent(email: str) -> None:
+ normalized = normalize_email(email)
+ assert normalize_email(normalized) == normalized
diff --git a/api/tests/unit_tests/services/test_account_service.py b/api/tests/unit_tests/services/test_account_service.py
index 75221d81f73..9bd00353a90 100644
--- a/api/tests/unit_tests/services/test_account_service.py
+++ b/api/tests/unit_tests/services/test_account_service.py
@@ -173,8 +173,20 @@ class TestAccountService:
assert persisted_account.timezone == "America/New_York"
assert persisted_account.last_login_ip == "203.0.113.10"
+ @pytest.mark.parametrize(
+ ("existing_email", "new_email", "normalized_email"),
+ [
+ ],
+ )
def test_create_account_rejects_normalized_email_only_when_requested(
self,
+ existing_email: str,
+ new_email: str,
+ normalized_email: str,
sqlite_session: Session,
mock_external_service_dependencies: _MockDependencies,
) -> None:
@@ -184,15 +196,15 @@ class TestAccountService:
sqlite_session.add(
Account(
name="Existing User",
- normalized_email="[email protected]",
+ email=existing_email,
+ normalized_email=normalized_email,
)
)
sqlite_session.commit()
with pytest.raises(AccountEmailAlreadyInUseError):
AccountService.create_account(
+ email=new_email,
name="New User",
interface_language="en-US",
check_normalized_email=True,
@@ -200,12 +212,12 @@ class TestAccountService:
)
duplicate = AccountService.create_account(
+ email=new_email,
name="New User",
interface_language="en-US",
session=sqlite_session,
)
- assert duplicate.normalized_email == "[email protected]"
+ assert duplicate.normalized_email == normalized_email
def test_create_account_uses_explicit_timezone(
self,