"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
fix(email): require a configured trusted authserv-id and From-domain alignment for SPF/DKIM
Lanre Shittua35b4b2ff01a
diff --git a/agent/src/channels/email.py b/agent/src/channels/email.pyindex 918fc3116..1c01f835d 100644--- a/agent/src/channels/email.py+++ b/agent/src/channels/email.py@@ -65,6 +65,15 @@ class EmailConfig(BaseModel):# Email authentication verification (anti-spoofing)verify_dkim: bool = True # Require Authentication-Results with dkim=passverify_spf: bool = True # Require Authentication-Results with spf=pass+ # authserv-id of the mail server this deployment's own inbox provider+ # stamps on delivery (the token right after "Authentication-Results:",+ # e.g. "mx.google.com") -- required to trust any spf=pass/dkim=pass+ # verdict at all. Without it there is no way to tell that header apart+ # from one an attacker forged in the message before it ever reached a+ # real authenticating server, so verification fails closed until it is+ # set. Find it by looking at the Authentication-Results header on any+ # genuine email already in the inbox.+ trusted_authserv_id: str = ""# Attachment handling — set allowed types to enable (e.g. ["application/pdf", "image/*"], or ["*"] for all)allowed_attachment_types: list[str] = Field(default_factory=list)@@ -157,6 +166,18 @@ class EmailChannel(BaseChannel):"Emails with spoofed From headers will be accepted. ""Set verify_dkim=true and verify_spf=true for anti-spoofing protection.")+ elif not self.config.trusted_authserv_id:+ self.logger.warning(+ "verify_spf/verify_dkim are on but trusted_authserv_id is not "+ "set, so no Authentication-Results header can be trusted "+ "(there is no way to tell your provider's real header apart "+ "from one an attacker forged before the message ever reached "+ "an authenticating server). Every inbound email will be "+ "rejected until you set trusted_authserv_id to the "+ "authserv-id your provider stamps -- copy it from the "+ "Authentication-Results header on any genuine email already "+ "in this inbox."+ )self.logger.info("Starting Email channel (IMAP polling mode)...")poll_seconds = max(5, int(self.config.poll_interval_seconds))@@ -471,11 +492,13 @@ class EmailChannel(BaseChannel):continue# --- Anti-spoofing: verify Authentication-Results ---- spf_pass, dkim_pass = self._check_authentication_results(parsed)+ spf_pass, dkim_pass = self._check_authentication_results(+ parsed, self.config.trusted_authserv_id+ )if self.config.verify_spf and not spf_pass:self.logger.warning(- "From {} rejected: SPF verification failed "- "(no 'spf=pass' in Authentication-Results header)",+ "From {} rejected: SPF verification failed (no aligned "+ "'spf=pass' in a trusted Authentication-Results header)",sender,)self._remember_processed_uid(uid, dedupe, cycle_uids)@@ -484,8 +507,8 @@ class EmailChannel(BaseChannel):continueif self.config.verify_dkim and not dkim_pass:self.logger.warning(- "From {} rejected: DKIM verification failed "- "(no 'dkim=pass' in Authentication-Results header)",+ "From {} rejected: DKIM verification failed (no aligned "+ "'dkim=pass' in a trusted Authentication-Results header)",sender,)self._remember_processed_uid(uid, dedupe, cycle_uids)@@ -827,26 +850,156 @@ class EmailChannel(BaseChannel):return cls._html_to_text(payload).strip()return payload.strip()+ # One ";"-separated resinfo clause: optional leading comment, the+ # method (optionally "/version"), "=", and the result token.+ _AR_CLAUSE_METHOD_RE = re.compile(+ r"(?:\(.*?\)\s*)?([a-zA-Z][a-zA-Z0-9_.-]*)(?:/[\d.]+)?\s*=\s*([a-zA-Z0-9_-]+)"+ )+ # A "ptype.property=value" pair inside a clause, e.g. smtp.mailfrom=...,+ # header.d=..., header.from=....+ _AR_PROPERTY_RE = re.compile(+ r"\b([a-zA-Z][a-zA-Z0-9_-]*)\.([a-zA-Z][a-zA-Z0-9_-]*)\s*=\s*([^\s;()]+)"+ )++ @classmethod+ def _select_trusted_authentication_results(+ cls, parsed_msg: Any, trusted_authserv_id: str+ ) -> str | None:+ """Return the one Authentication-Results header this deployment trusts.++ Requires ``trusted_authserv_id`` and returns the header whose leading+ authserv-id token matches it (case-insensitively); ``None`` when+ unconfigured or no header matches. An attacker cannot forge that+ match without already controlling the named server -- unlike+ trusting "whichever header is on top", which a message that never+ passed through a real authenticating server has no legitimate+ instance of at all, letting a forged one stand in for it.++ Args:+ parsed_msg: Parsed email message.+ trusted_authserv_id: This deployment's own mail server's+ authserv-id, or empty when not configured.++ Returns:+ The trusted header's raw value, or ``None``.+ """+ if not trusted_authserv_id:+ return None+ headers = parsed_msg.get_all("Authentication-Results") or []+ wanted = trusted_authserv_id.strip().lower()+ for header in headers:+ authserv_id = str(header).split(";", 1)[0].strip().lower()+ if authserv_id == wanted:+ return str(header)+ return None+@staticmethod- def _check_authentication_results(parsed_msg: Any) -> tuple[bool, bool]:- """Parse the Authentication-Results header for SPF and DKIM verdicts.+ def _parse_authentication_results_clauses(+ header_value: str,+ ) -> list[dict[str, Any]]:+ """Split one Authentication-Results header into its per-method clauses.- Only the topmost such header is trusted. Each receiving mail server- prepends its own Authentication-Results header on delivery, so the- first one in the parsed message is the one our own mailbox provider- stamped; anything below it sat in the message as it arrived at that- server and could be an attacker-forged header of the same name,- which an OR-across-every-occurrence check would wrongly honour.+ Args:+ header_value: One raw ``Authentication-Results`` header value.++ Returns:+ A list of ``{"method": str, "result": str, "properties":+ {(ptype, property): value}}`` dicts, one per ``;``-separated+ resinfo segment after the authserv-id. A segment with no+ parseable ``method=result`` is skipped rather than raising, so+ one garbled clause does not lose the other, well-formed ones.+ """+ _, _, rest = header_value.partition(";")+ clauses: list[dict[str, Any]] = []+ for segment in rest.split(";"):+ match = EmailChannel._AR_CLAUSE_METHOD_RE.search(segment)+ if not match:+ continue+ properties: dict[tuple[str, str], str] = {}+ for prop_match in EmailChannel._AR_PROPERTY_RE.finditer(segment):+ ptype = prop_match.group(1).lower()+ prop = prop_match.group(2).lower()+ properties[(ptype, prop)] = prop_match.group(3)+ clauses.append(+ {+ "method": match.group(1).lower(),+ "result": match.group(2).lower(),+ "properties": properties,+ }+ )+ return clauses++ @staticmethod+ def _domains_align(candidate: str, from_domain: str) -> bool:+ """Conservative same-or-subdomain check, relaxed-alignment style.++ Not a full RFC 7489 organizational-domain computation (that needs a+ public suffix list this module does not carry) -- a same-or-subdomain+ match in either direction, which is the common case for a single+ organization's mail infrastructure and never accepts an unrelated+ domain.+ """+ candidate = candidate.strip().lower().rstrip(".")+ from_domain = from_domain.strip().lower().rstrip(".")+ if not candidate or not from_domain:+ return False+ return (+ candidate == from_domain+ or candidate.endswith("." + from_domain)+ or from_domain.endswith("." + candidate)+ )++ @classmethod+ def _check_authentication_results(+ cls, parsed_msg: Any, trusted_authserv_id: str = ""+ ) -> tuple[bool, bool]:+ """Parse the trusted Authentication-Results header for SPF/DKIM verdicts.++ A ``pass`` result alone is not enough: it only says the named+ SPF/DKIM domain is legitimate, not that it is the domain the message+ claims to be from. An attacker can pass SPF/DKIM for their own+ domain while spoofing the visible From: header to look like someone+ else's. Each mechanism therefore also requires either an explicit+ ``dmarc=pass`` in the same trusted header (DMARC evaluation already+ performs this alignment check) or the mechanism's own authenticated+ domain to align with the message's From: domain.++ Args:+ parsed_msg: Parsed email message.+ trusted_authserv_id: This deployment's own mail server's+ authserv-id. Verification fails closed (returns+ ``(False, False)``) without it -- see+ :meth:`_select_trusted_authentication_results`.Returns:A tuple of (spf_pass, dkim_pass) booleans."""- headers = parsed_msg.get_all("Authentication-Results")- if not headers:+ header = cls._select_trusted_authentication_results(+ parsed_msg, trusted_authserv_id+ )+ if header is None:return False, False- ar_lower = str(headers[0]).lower()- spf_pass = bool(re.search(r"\bspf\s*=\s*pass\b", ar_lower))- dkim_pass = bool(re.search(r"\bdkim\s*=\s*pass\b", ar_lower))++ from_domain = parseaddr(parsed_msg.get("From", ""))[1].rsplit("@", 1)[-1]+ clauses = cls._parse_authentication_results_clauses(header)+ dmarc_pass = any(+ c["method"] == "dmarc" and c["result"] == "pass" for c in clauses+ )++ spf_pass = False+ dkim_pass = False+ for clause in clauses:+ if clause["method"] == "spf" and clause["result"] == "pass":+ candidate = clause["properties"].get(("smtp", "mailfrom")) or clause[+ "properties"+ ].get(("smtp", "helo"), "")+ candidate_domain = candidate.rsplit("@", 1)[-1] if candidate else ""+ if dmarc_pass or cls._domains_align(candidate_domain, from_domain):+ spf_pass = True+ elif clause["method"] == "dkim" and clause["result"] == "pass":+ candidate_domain = clause["properties"].get(("header", "d"), "")+ if dmarc_pass or cls._domains_align(candidate_domain, from_domain):+ dkim_pass = Truereturn spf_pass, dkim_pass@classmethoddiff --git a/agent/tests/test_email_authentication_results.py b/agent/tests/test_email_authentication_results.pyindex 708413af6..c74aa123f 100644--- a/agent/tests/test_email_authentication_results.py+++ b/agent/tests/test_email_authentication_results.py@@ -1,6 +1,22 @@-"""Only the topmost Authentication-Results header (added by our own mail-provider on delivery) may be trusted for SPF/DKIM verdicts. A lower one-could be an attacker-forged header present in the message as it arrived."""+"""Anti-spoofing verification of the Authentication-Results header.++Two things must both hold before spf=pass/dkim=pass is honoured:++1. The header trusted is the one this deployment's own mail provider+ stamped, identified by a configured ``trusted_authserv_id`` -- never+ "whichever header happens to be on top". A message that never passed+ through a real authenticating server has no legitimate header at all, so+ trusting position instead of identity lets an attacker's own forged+ header stand in for it. Without ``trusted_authserv_id`` configured,+ verification fails closed.++2. The authenticated domain (SPF's smtp.mailfrom/smtp.helo, or DKIM's+ header.d) aligns with the message's visible From: domain, or the same+ trusted header already carries an explicit dmarc=pass (which performs+ this same alignment check itself). Otherwise an attacker who legitimately+ passes SPF/DKIM for their own domain while spoofing the From: header to+ look like someone else's would still be honoured.+"""from __future__ import annotations@@ -14,49 +30,141 @@ def _parse(raw: bytes):return BytesParser(policy=policy.default).parsebytes(raw)-def test_forged_header_alone_is_not_trusted() -> None:+def test_unconfigured_trusted_authserv_id_fails_closed() -> None:+ """No trust anchor configured -- cannot tell a real header from a forged+ one, so verification must not honour any header, however clean it looks."""+ raw = (+ b"From: [email protected]\r\n"+ b"To: [email protected]\r\n"+ b"Subject: legit\r\n"+ b"Authentication-Results: mx.ourprovider.com; spf=pass [email protected]; "+ b"dkim=pass header.d=trusted-domain.com\r\n"+ b"Content-Type: text/plain\r\n"+ b"\r\n"+ b"hello\r\n"+ )+ spf_pass, dkim_pass = EmailChannel._check_authentication_results(_parse(raw), "")+ assert (spf_pass, dkim_pass) == (False, False)+++def test_forged_header_with_unmatched_authserv_id_is_not_trusted() -> None:+ """A header whose authserv-id does not match the configured one is not+ the deployment's own provider's stamp -- it is never even inspected."""raw = (b"From: [email protected]\r\n"b"To: [email protected]\r\n"b"Subject: forged\r\n"- b"Authentication-Results: attacker-controlled; spf=pass; dkim=pass\r\n"+ b"Authentication-Results: attacker-controlled; spf=pass [email protected]; "+ b"dkim=pass header.d=trusted-domain.com\r\n"b"Content-Type: text/plain\r\n"b"\r\n"b"please transfer funds\r\n")- spf_pass, dkim_pass = EmailChannel._check_authentication_results(_parse(raw))+ spf_pass, dkim_pass = EmailChannel._check_authentication_results(+ _parse(raw), "mx.ourprovider.com"+ )+ assert (spf_pass, dkim_pass) == (False, False)+++def test_genuine_pass_from_the_matching_trusted_header_is_honoured() -> None:+ raw = (+ b"From: [email protected]\r\n"+ b"To: [email protected]\r\n"+ b"Subject: legit\r\n"+ b"Authentication-Results: mx.ourprovider.com; spf=pass [email protected]; "+ b"dkim=pass header.d=trusted-domain.com\r\n"+ b"Content-Type: text/plain\r\n"+ b"\r\n"+ b"hello\r\n"+ )+ spf_pass, dkim_pass = EmailChannel._check_authentication_results(+ _parse(raw), "mx.ourprovider.com"+ )assert (spf_pass, dkim_pass) == (True, True)- # Documents the residual limit of a position-only check: with no real- # provider header above it, a lone header is still (correctly) the one- # trusted, same as before. The next test is the case this fix changes.-def test_real_header_wins_over_a_forged_header_below_it() -> None:+def test_matching_header_is_trusted_regardless_of_position() -> None:+ """A forged header above the real one must not shadow it -- the real+ provider's header is found by identity, not by being first. This is the+ case a position-only ("topmost") check gets wrong, and the case a+ self-hosted deployment with more than one mail hop depends on."""raw = (b"From: [email protected]\r\n"b"To: [email protected]\r\n"b"Subject: forged\r\n"- b"Authentication-Results: mx.ourprovider.com; spf=fail [email protected]; dkim=fail\r\n"- b"Authentication-Results: attacker-injected; spf=pass; dkim=pass\r\n"+ b"Authentication-Results: attacker-injected; spf=pass [email protected]; "+ b"dkim=pass header.d=trusted-domain.com\r\n"+ b"Authentication-Results: mx.ourprovider.com; spf=fail [email protected]; "+ b"dkim=fail header.d=trusted-domain.com\r\n"+ b"Content-Type: text/plain\r\n"+ b"\r\n"+ b"please transfer funds\r\n"+ )+ spf_pass, dkim_pass = EmailChannel._check_authentication_results(+ _parse(raw), "mx.ourprovider.com"+ )+ # The real (matching) header's verdict is fail, not the forged one's pass.+ assert (spf_pass, dkim_pass) == (False, False)+++def test_pass_for_an_unaligned_domain_is_not_honoured() -> None:+ """spf=pass/dkim=pass only says the named domain is legitimate, not that+ it is the domain the message claims to be from. An attacker who+ legitimately controls attacker.com can pass SPF/DKIM for attacker.com+ while spoofing the visible From: header to a victim's domain -- that+ must not be honoured as proof the From: domain is genuine."""+ raw = (+ b"From: [email protected]\r\n"+ b"To: [email protected]\r\n"+ b"Subject: spoofed display, genuine auth for a different domain\r\n"+ b"Authentication-Results: mx.ourprovider.com; spf=pass [email protected]; "+ b"dkim=pass header.d=attacker.com\r\n"b"Content-Type: text/plain\r\n"b"\r\n"b"please transfer funds\r\n")- spf_pass, dkim_pass = EmailChannel._check_authentication_results(_parse(raw))+ spf_pass, dkim_pass = EmailChannel._check_authentication_results(+ _parse(raw), "mx.ourprovider.com"+ )assert (spf_pass, dkim_pass) == (False, False)-def test_genuine_pass_from_the_topmost_header_is_honoured() -> None:+def test_subdomain_of_the_from_domain_is_aligned() -> None:+ """A relaxed-alignment match: mail.trusted-domain.com authenticating for+ a From: address at trusted-domain.com is the same organization."""+ raw = (+ b"From: [email protected]\r\n"+ b"To: [email protected]\r\n"+ b"Subject: legit, sent via a subdomain mailer\r\n"+ b"Authentication-Results: mx.ourprovider.com; spf=pass [email protected]; "+ b"dkim=pass header.d=mail.trusted-domain.com\r\n"+ b"Content-Type: text/plain\r\n"+ b"\r\n"+ b"hello\r\n"+ )+ spf_pass, dkim_pass = EmailChannel._check_authentication_results(+ _parse(raw), "mx.ourprovider.com"+ )+ assert (spf_pass, dkim_pass) == (True, True)+++def test_dmarc_pass_substitutes_for_domain_alignment() -> None:+ """An explicit dmarc=pass in the trusted header already performed its+ own alignment check -- spf=pass is honoured on its strength alone even+ though smtp.mailfrom differs in form from the From: address."""raw = (b"From: [email protected]\r\n"b"To: [email protected]\r\n"b"Subject: legit\r\n"- b"Authentication-Results: mx.ourprovider.com; spf=pass [email protected]; dkim=pass header.d=trusted-domain.com\r\n"+ b"Authentication-Results: mx.ourprovider.com; spf=pass [email protected]; "+ b"dkim=pass header.d=trusted-domain.com; dmarc=pass header.from=trusted-domain.com\r\n"b"Content-Type: text/plain\r\n"b"\r\n"b"hello\r\n")- spf_pass, dkim_pass = EmailChannel._check_authentication_results(_parse(raw))+ spf_pass, dkim_pass = EmailChannel._check_authentication_results(+ _parse(raw), "mx.ourprovider.com"+ )assert (spf_pass, dkim_pass) == (True, True)@@ -69,5 +177,25 @@ def test_no_header_is_not_trusted() -> None:b"\r\n"b"hi\r\n")- spf_pass, dkim_pass = EmailChannel._check_authentication_results(_parse(raw))+ spf_pass, dkim_pass = EmailChannel._check_authentication_results(+ _parse(raw), "mx.ourprovider.com"+ )assert (spf_pass, dkim_pass) == (False, False)+++def test_spf_and_dkim_are_independent_verdicts() -> None:+ """One mechanism passing and aligned must not make the other pass too."""+ raw = (+ b"From: [email protected]\r\n"+ b"To: [email protected]\r\n"+ b"Subject: spf only\r\n"+ b"Authentication-Results: mx.ourprovider.com; spf=pass [email protected]; "+ b"dkim=fail header.d=trusted-domain.com\r\n"+ b"Content-Type: text/plain\r\n"+ b"\r\n"+ b"hello\r\n"+ )+ spf_pass, dkim_pass = EmailChannel._check_authentication_results(+ _parse(raw), "mx.ourprovider.com"+ )+ assert (spf_pass, dkim_pass) == (True, False)