:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(security): match block rules against look-alike dots in email domains (#125927)
Daniel Marchukfee9132a5700
diff --git a/products/security/backend/logic/accounts.py b/products/security/backend/logic/accounts.pyindex 8fd20e554736..6cdb606078a6 100644--- a/products/security/backend/logic/accounts.py+++ b/products/security/backend/logic/accounts.py@@ -5,6 +5,7 @@ import uuidfrom typing import Literalfrom django.db.models import Q, QuerySet+from django.db.models.functions import Lowerfrom posthog.dataclasses import frozenfrom posthog.helpers.email_utils import EmailLookupHandler@@ -12,7 +13,7 @@ from posthog.models import Organization, OrganizationMembership, Userfrom .decisions import PROTECTED_DOMAINfrom .rules import EmailPattern-from .subjects import DOT_INSENSITIVE_DOMAINS+from .subjects import DOT_INSENSITIVE_DOMAINS, HIDDEN_CHARACTERS, LOOKALIKE_DOTS# An email root, domain or pattern count has no index to use, so an uncapped count would scan# every row of posthog_user while the admin waits.@@ -84,37 +85,60 @@ def _capped(queryset: QuerySet) -> CappedCount:return CappedCount(count=min(found, COUNT_CAP), capped=found > COUNT_CAP)+_DOT = f"[.{LOOKALIKE_DOTS}]"+_HIDDEN = f"[{HIDDEN_CHARACTERS}]*"+++def _lookalike(literal: str) -> str:+ """Matches the literal as a decision reads it: any dot may be a look-alike, and hidden characters may sit anywhere."""+ return _HIDDEN + _HIDDEN.join(_DOT if char == "." else re.escape(char) for char in literal) + _HIDDEN+++def _email_regex(address: str) -> str:+ return f"^{_lookalike(address)}$"++def _email_root_regex(root: str) -> str:"""Every stored address that folds to this root."""local, _, domain = root.rpartition("@")suffix = r"(\+[^@]*)?"if domain in DOT_INSENSITIVE_DOMAINS:- dotted_local = r"\.*".join(re.escape(char) for char in local)- return rf"^{dotted_local}{suffix}@(gmail|googlemail)\.com$"- return rf"^{re.escape(local)}{suffix}@{re.escape(domain)}$"+ dotted_local = f"[.{LOOKALIKE_DOTS}{HIDDEN_CHARACTERS}]*".join(re.escape(char) for char in local)+ return rf"^{_HIDDEN}{dotted_local}{_HIDDEN}{suffix}@({_lookalike('gmail.com')}|{_lookalike('googlemail.com')})$"+ return rf"^{_lookalike(local)}{suffix}@{_lookalike(domain)}$"+++def _email_domain_regex(domain: str) -> str:+ """Every stored address a block on this domain refuses."""+ return f"(@|{_DOT}){_lookalike(domain)}$"def _email_pattern_regex(pattern: EmailPattern) -> str:- return "^" + ".*".join(re.escape(segment) for segment in pattern.segments) + "$"+ return "^" + ".*".join(_lookalike(segment) for segment in pattern.segments) + "$"def count_active_accounts(target_type: CountableType, value: str) -> CappedCount:- if target_type == "email":- # Matches resolve_subject's fold (LOWER via EmailLookupHandler), not `iexact` (UPPER):- # the two fold Unicode differently, so a count on one fold could disagree with who resolves.- return _capped(EmailLookupHandler.users_matching_email(value, User.objects.filter(is_active=True)))+ active = User.objects.filter(is_active=True)match target_type:+ case "email":+ # Matches resolve_subject's fold (LOWER via EmailLookupHandler), not `iexact` (UPPER):+ # the two fold Unicode differently, so a count on one fold could disagree with who resolves.+ exact = EmailLookupHandler.users_matching_email(value, active).values("pk")+ lookalikes = active.alias(email_lower=Lower("email")).filter(+ email_lower__regex=_email_regex(value.strip().lower())+ )+ return _capped(active.filter(Q(pk__in=exact) | Q(pk__in=lookalikes.values("pk"))))case "email_root":condition = Q(email__iregex=_email_root_regex(value))case "email_domain":- condition = Q(email__iendswith=f"@{value}") | Q(email__iendswith=f".{value}")+ condition = Q(email__iregex=_email_domain_regex(value))case "email_pattern":pattern = EmailPattern.parse(value.strip().lower())# A pattern the snapshot would refuse matches nobody at decision time, so it counts nobody.if pattern is None:return CappedCount(count=0, capped=False)condition = Q(email__iregex=_email_pattern_regex(pattern))- return _capped(User.objects.filter(condition, is_active=True))+ return _capped(active.filter(condition))def count_active_org_members(organization_id: str) -> tuple[bool, CappedCount]:diff --git a/products/security/backend/logic/decisions.py b/products/security/backend/logic/decisions.pyindex 1523332a7a45..aee64d516e8a 100644--- a/products/security/backend/logic/decisions.py+++ b/products/security/backend/logic/decisions.py@@ -51,13 +51,16 @@ def deciding_rule(index: RuleIndex, subject: Subject, surface: Surface, now: datif effect == "block" and is_protected_domain(subject.domain):return Noneallowed_targets = SURFACE_TARGET_TYPES.get(surface)- for rule in candidates(index, subject):- if (- rule.effect == effect- and (allowed_targets is None or rule.target_type in allowed_targets)- and surface in SCOPE_SURFACES.get(rule.scope, frozenset())- and _is_active(rule, now)- and rule_matches(rule, subject)- ):- return rule- return None+ # Only blocks read the look-alike form: an exemption for one mailbox must not cover a different spelling.+ variants = [subject] if effect != "block" or subject.lookalike is None else [subject, subject.lookalike]+ matched = [+ rule+ for variant in variants+ for rule in candidates(index, variant)+ if rule.effect == effect+ and (allowed_targets is None or rule.target_type in allowed_targets)+ and surface in SCOPE_SURFACES.get(rule.scope, frozenset())+ and _is_active(rule, now)+ and rule_matches(rule, variant)+ ]+ return min(matched, key=lambda rule: rule.id, default=None)diff --git a/products/security/backend/logic/subjects.py b/products/security/backend/logic/subjects.pyindex fd5380c9ddbf..664dc0c77f65 100644--- a/products/security/backend/logic/subjects.py+++ b/products/security/backend/logic/subjects.py@@ -2,6 +2,7 @@import uuidimport ipaddress+import dataclassesfrom collections.abc import Iterablefrom posthog.dataclasses import frozen@@ -12,6 +13,10 @@ DOT_INSENSITIVE_DOMAINS = frozenset({"gmail.com", "googlemail.com"})# SMTP caps a deliverable address at 254 characters. Longer input is refused so the domain suffix expansion stays small.MAX_ADDRESS_LENGTH = 254+LOOKALIKE_DOTS = "\u3002\uff0e\uff61\ufe52\u2024"+HIDDEN_CHARACTERS = "\u00ad\u200b\u200c\u200d\u2060\ufeff"+_FOLD_LOOKALIKES = str.maketrans(dict.fromkeys(LOOKALIKE_DOTS, ".") | dict.fromkeys(HIDDEN_CHARACTERS, None))+@frozenclass Subject:@@ -21,6 +26,11 @@ class Subject:user_uuid: str | None = Noneorganization_ids: frozenset[str] = frozenset()ip: ipaddress.IPv4Address | ipaddress.IPv6Address | None = None+ lookalike: "Subject | None" = None+++def fold_lookalikes(value: str) -> str:+ return value.translate(_FOLD_LOOKALIKES)def fold_email_root(address: str) -> str | None:@@ -71,12 +81,24 @@ def normalize_subject(subject_domain = (domain or "").strip().lower().removeprefix("@") or Noneif subject_domain and len(subject_domain) > MAX_ADDRESS_LENGTH:subject_domain = None- organizations = frozenset(filter(None, (_canonical_uuid(org) for org in organization_ids)))- return Subject(+ subject = Subject(email=address,email_root=fold_email_root(address) if address else None,domain=subject_domain,user_uuid=_canonical_uuid(user_uuid),- organization_ids=organizations,+ organization_ids=frozenset(filter(None, (_canonical_uuid(org) for org in organization_ids))),ip=_parse_ip(ip),)+ folded_address = fold_lookalikes(address) if address else None+ folded_domain = fold_lookalikes(subject_domain) if subject_domain else None+ if folded_address == address and folded_domain == subject_domain:+ return subject+ return dataclasses.replace(+ subject,+ lookalike=dataclasses.replace(+ subject,+ email=folded_address,+ email_root=fold_email_root(folded_address) if folded_address else None,+ domain=folded_domain or None,+ ),+ )diff --git a/products/security/backend/tests/test_accounts.py b/products/security/backend/tests/test_accounts.pyindex 1820e37e9bf1..fcd6ee98be21 100644--- a/products/security/backend/tests/test_accounts.py+++ b/products/security/backend/tests/test_accounts.py@@ -53,12 +53,18 @@ class TestAccounts(BaseTest):self._user("[email protected]")self._user("[email protected]", active=False)self._user("[email protected]")+ self._user("y@sub\u3002throwaway\uff0eexample")+ self._user("z@throw\u200baway.example")+ self._user("[email protected]")+ self._user("farm\u200bbot@example\u3002com")+ self._user("farm\[email protected]")+ self._user("d\u3002o.t@gmail\uff0ecom")self._user("[email protected]")- assert count_active_accounts("email_root", "[email protected]").count == 1- assert count_active_accounts("email", "[email protected]").count == 1- assert count_active_accounts("email_domain", "throwaway.example").count == 1- assert count_active_accounts("email_root", "[email protected]").count == 1- assert count_active_accounts("email_pattern", "farm.bot+*@example.com").count == 1+ assert count_active_accounts("email_root", "[email protected]").count == 2+ assert count_active_accounts("email", "[email protected]").count == 2+ assert count_active_accounts("email_domain", "throwaway.example").count == 3+ assert count_active_accounts("email_root", "[email protected]").count == 2+ assert count_active_accounts("email_pattern", "farm.bot+*@example.com").count == 2assert count_active_accounts("email_pattern", "far.*@*").count == 0assert count_active_accounts("email_pattern", "*@*").count == 0diff --git a/products/security/backend/tests/test_decisions.py b/products/security/backend/tests/test_decisions.pyindex 711055252244..a60728218317 100644--- a/products/security/backend/tests/test_decisions.py+++ b/products/security/backend/tests/test_decisions.py@@ -174,3 +174,56 @@ class TestSubjectLength(TestCase):def test_overlong_input_carries_no_email_or_domain(self, _name: str, kwargs: dict[str, str]) -> None:subject = normalize_subject(**kwargs)assert subject.email is None and subject.domain is None+++def _block_rule(target_type: str, target_value: str) -> SnapshotRule:+ rule = SnapshotRule.from_wire(+ {"id": "a", "targetType": target_type, "targetValue": target_value, "effect": "block", "scope": "all_access"}+ )+ assert rule is not None+ return rule+++class TestLookalikeAddresses(TestCase):+ @parameterized.expand(+ [+ ("fullwidth dot", "email_domain", "example.com", "user@sub\uff0eexample.com", True),+ ("ideographic dot", "email_domain", "example.com", "user@sub\u3002example.com", True),+ ("halfwidth ideographic dot", "email_domain", "example.com", "user@sub\uff61example\uff61com", True),+ ("small full stop", "email_domain", "example.com", "user@example\ufe52com", True),+ ("one dot leader", "email_domain", "example.com", "user@example\u2024com", True),+ ("zero-width space in the domain", "email_domain", "example.com", "user@exa\u200bmple.com", True),+ ("zero-width space in the address", "email", "[email protected]", "abu\[email protected]", True),+ ("pattern", "email_pattern", "abuser*@example.com", "abuser7@example\u3002com", True),+ ("email root", "email_root", "[email protected]", "farm\[email protected]", True),+ ("gmail root", "email_root", "[email protected]", "d\u3002o.t@gmail\uff0ecom", True),+ ("ascii address is unchanged", "email_domain", "example.com", "[email protected]", False),+ ("shared suffix is not a subdomain", "email_domain", "example.com", "user@not\u3002example.org", False),+ ]+ )+ def test_block_rule_reads_lookalikes_as_ascii(+ self, _name: str, target_type: str, target_value: str, email: str, blocked: bool+ ) -> None:+ index = build_index([_block_rule(target_type, target_value)])+ decided = deciding_rule(index, normalize_subject(email=email), Surface.APP, NOW)+ assert (decided is not None) is blocked++ def test_exemption_covers_only_the_exact_spelling(self) -> None:+ rule = SnapshotRule.from_wire(+ {+ "id": "a",+ "targetType": "email",+ "targetValue": "[email protected]",+ "effect": "exempt",+ "scope": "email_code",+ }+ )+ assert rule is not None+ index = build_index([rule])+ assert deciding_rule(index, normalize_subject(email="[email protected]"), Surface.EMAIL_CODE, NOW) is not None+ assert deciding_rule(index, normalize_subject(email="o\[email protected]"), Surface.EMAIL_CODE, NOW) is None++ def test_lookalike_protected_domain_is_not_protected(self) -> None:+ index = build_index([_block_rule("everyone", "")])+ assert deciding_rule(index, normalize_subject(email="[email protected]"), Surface.APP, NOW) is None+ assert deciding_rule(index, normalize_subject(email="staff@posthog\u3002com"), Surface.APP, NOW) is not None