Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(security): match block rules against look-alike dots in email domains (#125927)

Daniel Marchukfee9132a5700
diff --git a/products/security/backend/logic/accounts.py b/products/security/backend/logic/accounts.py
index 8fd20e554736..6cdb606078a6 100644
--- a/products/security/backend/logic/accounts.py
+++ b/products/security/backend/logic/accounts.py
@@ -5,6 +5,7 @@ import uuid
from typing import Literal
from django.db.models import Q, QuerySet
+from django.db.models.functions import Lower
from posthog.dataclasses import frozen
from posthog.helpers.email_utils import EmailLookupHandler
@@ -12,7 +13,7 @@ from posthog.models import Organization, OrganizationMembership, User
from .decisions import PROTECTED_DOMAIN
from .rules import EmailPattern
-from .subjects import DOT_INSENSITIVE_DOMAINS
+from .subjects import DOT_INSENSITIVE_DOMAINS, HIDDEN_CHARACTERS, LOOKALIKE_DOTS
# An email root, domain or pattern count has no index to use, so an uncapped count would scan
# every row of posthog_user while the admin waits.
@@ -84,37 +85,60 @@ def _capped(queryset: QuerySet) -> CappedCount:
return CappedCount(count=min(found, COUNT_CAP), capped=found > COUNT_CAP)
+_DOT = f"[.{LOOKALIKE_DOTS}]"
+_HIDDEN = f"[{HIDDEN_CHARACTERS}]*"
+
+
+def _lookalike(literal: str) -> str:
+ """Matches the literal as a decision reads it: any dot may be a look-alike, and hidden characters may sit anywhere."""
+ return _HIDDEN + _HIDDEN.join(_DOT if char == "." else re.escape(char) for char in literal) + _HIDDEN
+
+
+def _email_regex(address: str) -> str:
+ return f"^{_lookalike(address)}$"
+
+
def _email_root_regex(root: str) -> str:
"""Every stored address that folds to this root."""
local, _, domain = root.rpartition("@")
suffix = r"(\+[^@]*)?"
if domain in DOT_INSENSITIVE_DOMAINS:
- dotted_local = r"\.*".join(re.escape(char) for char in local)
- return rf"^{dotted_local}{suffix}@(gmail|googlemail)\.com$"
- return rf"^{re.escape(local)}{suffix}@{re.escape(domain)}$"
+ dotted_local = f"[.{LOOKALIKE_DOTS}{HIDDEN_CHARACTERS}]*".join(re.escape(char) for char in local)
+ return rf"^{_HIDDEN}{dotted_local}{_HIDDEN}{suffix}@({_lookalike('gmail.com')}|{_lookalike('googlemail.com')})$"
+ return rf"^{_lookalike(local)}{suffix}@{_lookalike(domain)}$"
+
+
+def _email_domain_regex(domain: str) -> str:
+ """Every stored address a block on this domain refuses."""
+ return f"(@|{_DOT}){_lookalike(domain)}$"
def _email_pattern_regex(pattern: EmailPattern) -> str:
- return "^" + ".*".join(re.escape(segment) for segment in pattern.segments) + "$"
+ return "^" + ".*".join(_lookalike(segment) for segment in pattern.segments) + "$"
def count_active_accounts(target_type: CountableType, value: str) -> CappedCount:
- if target_type == "email":
- # Matches resolve_subject's fold (LOWER via EmailLookupHandler), not `iexact` (UPPER):
- # the two fold Unicode differently, so a count on one fold could disagree with who resolves.
- return _capped(EmailLookupHandler.users_matching_email(value, User.objects.filter(is_active=True)))
+ active = User.objects.filter(is_active=True)
match target_type:
+ case "email":
+ # Matches resolve_subject's fold (LOWER via EmailLookupHandler), not `iexact` (UPPER):
+ # the two fold Unicode differently, so a count on one fold could disagree with who resolves.
+ exact = EmailLookupHandler.users_matching_email(value, active).values("pk")
+ lookalikes = active.alias(email_lower=Lower("email")).filter(
+ email_lower__regex=_email_regex(value.strip().lower())
+ )
+ return _capped(active.filter(Q(pk__in=exact) | Q(pk__in=lookalikes.values("pk"))))
case "email_root":
condition = Q(email__iregex=_email_root_regex(value))
case "email_domain":
- condition = Q(email__iendswith=f"@{value}") | Q(email__iendswith=f".{value}")
+ condition = Q(email__iregex=_email_domain_regex(value))
case "email_pattern":
pattern = EmailPattern.parse(value.strip().lower())
# A pattern the snapshot would refuse matches nobody at decision time, so it counts nobody.
if pattern is None:
return CappedCount(count=0, capped=False)
condition = Q(email__iregex=_email_pattern_regex(pattern))
- return _capped(User.objects.filter(condition, is_active=True))
+ return _capped(active.filter(condition))
def count_active_org_members(organization_id: str) -> tuple[bool, CappedCount]:
diff --git a/products/security/backend/logic/decisions.py b/products/security/backend/logic/decisions.py
index 1523332a7a45..aee64d516e8a 100644
--- a/products/security/backend/logic/decisions.py
+++ b/products/security/backend/logic/decisions.py
@@ -51,13 +51,16 @@ def deciding_rule(index: RuleIndex, subject: Subject, surface: Surface, now: dat
if effect == "block" and is_protected_domain(subject.domain):
return None
allowed_targets = SURFACE_TARGET_TYPES.get(surface)
- for rule in candidates(index, subject):
- if (
- rule.effect == effect
- and (allowed_targets is None or rule.target_type in allowed_targets)
- and surface in SCOPE_SURFACES.get(rule.scope, frozenset())
- and _is_active(rule, now)
- and rule_matches(rule, subject)
- ):
- return rule
- return None
+ # Only blocks read the look-alike form: an exemption for one mailbox must not cover a different spelling.
+ variants = [subject] if effect != "block" or subject.lookalike is None else [subject, subject.lookalike]
+ matched = [
+ rule
+ for variant in variants
+ for rule in candidates(index, variant)
+ if rule.effect == effect
+ and (allowed_targets is None or rule.target_type in allowed_targets)
+ and surface in SCOPE_SURFACES.get(rule.scope, frozenset())
+ and _is_active(rule, now)
+ and rule_matches(rule, variant)
+ ]
+ return min(matched, key=lambda rule: rule.id, default=None)
diff --git a/products/security/backend/logic/subjects.py b/products/security/backend/logic/subjects.py
index fd5380c9ddbf..664dc0c77f65 100644
--- a/products/security/backend/logic/subjects.py
+++ b/products/security/backend/logic/subjects.py
@@ -2,6 +2,7 @@
import uuid
import ipaddress
+import dataclasses
from collections.abc import Iterable
from posthog.dataclasses import frozen
@@ -12,6 +13,10 @@ DOT_INSENSITIVE_DOMAINS = frozenset({"gmail.com", "googlemail.com"})
# SMTP caps a deliverable address at 254 characters. Longer input is refused so the domain suffix expansion stays small.
MAX_ADDRESS_LENGTH = 254
+LOOKALIKE_DOTS = "\u3002\uff0e\uff61\ufe52\u2024"
+HIDDEN_CHARACTERS = "\u00ad\u200b\u200c\u200d\u2060\ufeff"
+_FOLD_LOOKALIKES = str.maketrans(dict.fromkeys(LOOKALIKE_DOTS, ".") | dict.fromkeys(HIDDEN_CHARACTERS, None))
+
@frozen
class Subject:
@@ -21,6 +26,11 @@ class Subject:
user_uuid: str | None = None
organization_ids: frozenset[str] = frozenset()
ip: ipaddress.IPv4Address | ipaddress.IPv6Address | None = None
+ lookalike: "Subject | None" = None
+
+
+def fold_lookalikes(value: str) -> str:
+ return value.translate(_FOLD_LOOKALIKES)
def fold_email_root(address: str) -> str | None:
@@ -71,12 +81,24 @@ def normalize_subject(
subject_domain = (domain or "").strip().lower().removeprefix("@") or None
if subject_domain and len(subject_domain) > MAX_ADDRESS_LENGTH:
subject_domain = None
- organizations = frozenset(filter(None, (_canonical_uuid(org) for org in organization_ids)))
- return Subject(
+ subject = Subject(
email=address,
email_root=fold_email_root(address) if address else None,
domain=subject_domain,
user_uuid=_canonical_uuid(user_uuid),
- organization_ids=organizations,
+ organization_ids=frozenset(filter(None, (_canonical_uuid(org) for org in organization_ids))),
ip=_parse_ip(ip),
)
+ folded_address = fold_lookalikes(address) if address else None
+ folded_domain = fold_lookalikes(subject_domain) if subject_domain else None
+ if folded_address == address and folded_domain == subject_domain:
+ return subject
+ return dataclasses.replace(
+ subject,
+ lookalike=dataclasses.replace(
+ subject,
+ email=folded_address,
+ email_root=fold_email_root(folded_address) if folded_address else None,
+ domain=folded_domain or None,
+ ),
+ )
diff --git a/products/security/backend/tests/test_accounts.py b/products/security/backend/tests/test_accounts.py
index 1820e37e9bf1..fcd6ee98be21 100644
--- a/products/security/backend/tests/test_accounts.py
+++ b/products/security/backend/tests/test_accounts.py
@@ -53,12 +53,18 @@ class TestAccounts(BaseTest):
self._user("[email protected]")
self._user("[email protected]", active=False)
self._user("[email protected]")
+ self._user("y@sub\u3002throwaway\uff0eexample")
+ self._user("z@throw\u200baway.example")
+ self._user("[email protected]")
+ self._user("farm\u200bbot@example\u3002com")
+ self._user("farm\[email protected]")
+ self._user("d\u3002o.t@gmail\uff0ecom")
self._user("[email protected]")
- assert count_active_accounts("email_root", "[email protected]").count == 1
- assert count_active_accounts("email", "[email protected]").count == 1
- assert count_active_accounts("email_domain", "throwaway.example").count == 1
- assert count_active_accounts("email_root", "[email protected]").count == 1
- assert count_active_accounts("email_pattern", "farm.bot+*@example.com").count == 1
+ assert count_active_accounts("email_root", "[email protected]").count == 2
+ assert count_active_accounts("email", "[email protected]").count == 2
+ assert count_active_accounts("email_domain", "throwaway.example").count == 3
+ assert count_active_accounts("email_root", "[email protected]").count == 2
+ assert count_active_accounts("email_pattern", "farm.bot+*@example.com").count == 2
assert count_active_accounts("email_pattern", "far.*@*").count == 0
assert count_active_accounts("email_pattern", "*@*").count == 0
diff --git a/products/security/backend/tests/test_decisions.py b/products/security/backend/tests/test_decisions.py
index 711055252244..a60728218317 100644
--- a/products/security/backend/tests/test_decisions.py
+++ b/products/security/backend/tests/test_decisions.py
@@ -174,3 +174,56 @@ class TestSubjectLength(TestCase):
def test_overlong_input_carries_no_email_or_domain(self, _name: str, kwargs: dict[str, str]) -> None:
subject = normalize_subject(**kwargs)
assert subject.email is None and subject.domain is None
+
+
+def _block_rule(target_type: str, target_value: str) -> SnapshotRule:
+ rule = SnapshotRule.from_wire(
+ {"id": "a", "targetType": target_type, "targetValue": target_value, "effect": "block", "scope": "all_access"}
+ )
+ assert rule is not None
+ return rule
+
+
+class TestLookalikeAddresses(TestCase):
+ @parameterized.expand(
+ [
+ ("fullwidth dot", "email_domain", "example.com", "user@sub\uff0eexample.com", True),
+ ("ideographic dot", "email_domain", "example.com", "user@sub\u3002example.com", True),
+ ("halfwidth ideographic dot", "email_domain", "example.com", "user@sub\uff61example\uff61com", True),
+ ("small full stop", "email_domain", "example.com", "user@example\ufe52com", True),
+ ("one dot leader", "email_domain", "example.com", "user@example\u2024com", True),
+ ("zero-width space in the domain", "email_domain", "example.com", "user@exa\u200bmple.com", True),
+ ("zero-width space in the address", "email", "[email protected]", "abu\[email protected]", True),
+ ("pattern", "email_pattern", "abuser*@example.com", "abuser7@example\u3002com", True),
+ ("email root", "email_root", "[email protected]", "farm\[email protected]", True),
+ ("gmail root", "email_root", "[email protected]", "d\u3002o.t@gmail\uff0ecom", True),
+ ("ascii address is unchanged", "email_domain", "example.com", "[email protected]", False),
+ ("shared suffix is not a subdomain", "email_domain", "example.com", "user@not\u3002example.org", False),
+ ]
+ )
+ def test_block_rule_reads_lookalikes_as_ascii(
+ self, _name: str, target_type: str, target_value: str, email: str, blocked: bool
+ ) -> None:
+ index = build_index([_block_rule(target_type, target_value)])
+ decided = deciding_rule(index, normalize_subject(email=email), Surface.APP, NOW)
+ assert (decided is not None) is blocked
+
+ def test_exemption_covers_only_the_exact_spelling(self) -> None:
+ rule = SnapshotRule.from_wire(
+ {
+ "id": "a",
+ "targetType": "email",
+ "targetValue": "[email protected]",
+ "effect": "exempt",
+ "scope": "email_code",
+ }
+ )
+ assert rule is not None
+ index = build_index([rule])
+ assert deciding_rule(index, normalize_subject(email="[email protected]"), Surface.EMAIL_CODE, NOW) is not None
+ assert deciding_rule(index, normalize_subject(email="o\[email protected]"), Surface.EMAIL_CODE, NOW) is None
+
+ def test_lookalike_protected_domain_is_not_protected(self) -> None:
+ index = build_index([_block_rule("everyone", "")])
+ assert deciding_rule(index, normalize_subject(email="[email protected]"), Surface.APP, NOW) is None
+ assert deciding_rule(index, normalize_subject(email="staff@posthog\u3002com"), Surface.APP, NOW) is not None